From 6b89cf47f0f18a572a461a7ccd56bdc304593bb3 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 19 Oct 2026 13:56:14 +0300 Subject: [PATCH 01/97] docs(docs): record the wave 3 steps 1-8 merge PR #95 merged to main as 6d2a7372 at 22:44 UTC on 2026-10-18. Record the merge checkpoint in the Phase 2.5.5 register, point current.md and the agent guides at Step 9, and keep Wave 3 open: Steps 9-19 are not started. - phase register: merge checkpoint (completed scope, review attribution, the independent review's dispositions, publication state) and the maintainer-approved 9a-9d split of Step 9 as rows 9a-9d of the increment table - deferred-tasks: the independent review's unactioned findings, and the SonarCloud main-branch gate recorded as done - limits stated rather than implied: a3e164b5 has author verification only; CI for the merged head and the merge commit finished green after the merge; the SonarCloud dispositions were applied on main on 2026-10-19 Refs: ADR-0111 Co-Authored-By: Claude Sonnet 5.5 --- AGENTS.md | 16 ++- CLAUDE.md | 16 ++- docs/roadmap/current.md | 85 +++++------ docs/roadmap/deferred-tasks.md | 71 +++++++++- .../phase-2.5.5-hardening-and-remediation.md | 133 ++++++++++++++++++ 5 files changed, 268 insertions(+), 53 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8d7dc760..6b36255f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -115,12 +115,16 @@ failed reviewer-preparation records and source/native/platform limits remain explicit; source CI does not certify later documentation commits. The [acceptance register](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) joins exact evidence and historical corrections. Steps 1–8 are accepted; -Steps 9–19 have NOT STARTED. After the private handoff, the maintainer authorized -an initial PR for completed Steps 1–8. This is a scoped publication checkpoint; -whole-wave acceptance remains Step 19. Later work stays with the next agent. -Check that PR's status before committing Step 9 on `development`, because new -commits extend its review scope. No provider, credential, billing or new ADR -approval is pending for the approved implementation plan. +**Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 +itself is NOT complete: Steps 9–19 have NOT STARTED and whole-wave acceptance remains +Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) +records what merged, its review attributions, and one open limit: the final head `a3e164b5` +(corrections from a further independent review, including a secret-scrubbing rule) has +author verification only. CI for it and the merge commit finished green, and SonarCloud's +`main` gate (red since at least PR #90) was cleared on 2026-10-19. Next is Step 9 (release +map/custody substrate, #46/ADR-0111, split into 9a–9d in the increment table); new commits +on `development` start a new publication checkpoint. No provider, +credential, billing or new ADR approval is pending for the approved implementation plan. See [docs/roadmap/current.md](docs/roadmap/current.md) for live status. diff --git a/CLAUDE.md b/CLAUDE.md index 8c828f2a..30f8ce78 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -204,12 +204,16 @@ failed reviewer-preparation records and source/native/platform limits remain explicit; source CI does not certify later documentation commits. The [acceptance register](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) joins exact evidence and historical corrections. Steps 1–8 are accepted; -Steps 9–19 have NOT STARTED. After the private handoff, the maintainer authorized -an initial PR for completed Steps 1–8. This is a scoped publication checkpoint; -whole-wave acceptance remains Step 19. Later work stays with the next agent. -Check that PR's status before committing Step 9 on `development`, because new -commits extend its review scope. No provider, credential, billing or new ADR -approval is pending for the approved implementation plan. +**Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 +itself is NOT complete: Steps 9–19 have NOT STARTED and whole-wave acceptance remains +Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) +records what merged, its review attributions, and one open limit: the final head `a3e164b5` +(corrections from a further independent review, including a secret-scrubbing rule) has +author verification only. CI for it and the merge commit finished green, and SonarCloud's +`main` gate (red since at least PR #90) was cleared on 2026-10-19. Next is Step 9 (release +map/custody substrate, #46/ADR-0111, split into 9a–9d in the increment table); new commits +on `development` start a new publication checkpoint. No provider, +credential, billing or new ADR approval is pending for the approved implementation plan. For live status, per-PR history, milestone dates, and open obligations, see the canonical home [docs/roadmap/current.md](docs/roadmap/current.md); diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index 4e3cec4d..f8bedd2b 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -1,7 +1,7 @@ # Current state > Status: Living -> Last updated: 2026-10-18 +> Last updated: 2026-10-19 - **Related**: [README.md](README.md), [phases/phase-2.5-cli-consolidation.md](phases/phase-2.5-cli-consolidation.md), [phases/phase-2.5.5-hardening-and-remediation.md](phases/phase-2.5.5-hardening-and-remediation.md), [phases/phase-2-cli.md](phases/phase-2-cli.md), [deferred-tasks.md](deferred-tasks.md), [../project-structure.md](../project-structure.md), [../tech-stack.md](../tech-stack.md) @@ -134,18 +134,37 @@ future live headers and existing approved residuals. Wave 2 has no remaining imp review, publication, ADR approval, provider capture, credential, billing or user-decision gate. The [deferred register](deferred-tasks.md) retains its existing limits; closure adds no deferral. +## Wave 3 Steps 1–8 merged — PR #95 + +**Wave 3 Steps 1–8 merged to `main` on 2026-10-18 as [PR #95](https://github.com/HodeTech/Relavium/pull/95), +merge commit `6d2a7372` (22:44 UTC). Wave 3 itself is not complete: Steps 9–19 have NOT STARTED and +whole-wave acceptance remains Step 19.** The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) +joins the completed scope, every review attribution, the independent review's dispositions and the +publication state observed at merge. It was the maintainer-authorized, scoped publication checkpoint. + +On `main` now: close handling, safe source labels and exhaustiveness guards (Step 1); shared admission and +diagnostics (2); secret provenance at public sinks, ADR-0113 (3); the typed error/envelope substrate (4); tool and +engine tails (5); the pure process/discovery substrate, ADR-0110 (6); the lazy runtime, finite compiled graph and +exact-tarball consumers, ADR-0111 (7); and creation-owned native process integration (8). + +Three points are recorded rather than papered over; two are now resolved. **Open:** the merged head `a3e164b5` carries +corrections from a further independent review (one medium and three low findings: a secret-scrubbing production rule and +two behaviour-neutral import moves, plus tests and causal-control pins) that have **author verification only** — no +independent review rounds and no security sitting cover them. **Resolved:** CI had not finished when the merge happened +(the required `lint · typecheck · test` job was still running); the three runs later completed `success`, 15 of 15 jobs +each, as the [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#publication-state-observed-at-merge) +records. And SonarCloud's quality gate, which had failed on `main` since at least PR #90 because its dispositions lived +only on PR branches, was cleared on 2026-10-19 (zero open vulnerabilities, gate OK). The review's remaining low and nit +findings are unactioned, in [deferred-tasks.md](deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18). + ## Next work — Wave 3 — Clear the ground -Wave 3 implementation is in progress on `development`; Steps 1–8 have scoped -independent acceptance. The initial publication checkpoint is -[PR #95](https://github.com/HodeTech/Relavium/pull/95). A second maintainer systematic -review at `a5a5d45e` reopened corrective work; its -[current correction register](phases/phase-2.5.5-hardening-and-remediation.md#pr-95-second-systematic-review-corrections--2026-10-18) -owns dispositions and remaining gates. Corrective source reviews are complete; -the two later native test-fixture increments have [separate scoped acceptance](phases/phase-2.5.5-hardening-and-remediation.md#pr-95-supplemental-native-fixture-source-acceptance--2026-10-18). -Final pushed-head publication checks remain required, with their live results on -PR #95. Steps 9–19 have not started; check the PR status before extending its scope -with Step 9. See the [Wave 3 queue](#wave-3--clear-the-ground). +Wave 3 implementation continues on `development`: Steps 1–8 are merged, **Step 9 is next** — the release +map/custody substrate (#46, ADR-0111), split into 9a–9d in the [increment table](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments), +which carries a dedicated archive/crypto security sitting. Steps 10–19 +follow the approved [nineteen-increment plan](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) +and end with whole-wave acceptance at Step 19. New implementation commits start a new publication checkpoint; PR #95 +no longer bounds their review scope. See the [Wave 3 queue](#wave-3--clear-the-ground). ## Historical W7 implementation checkpoints @@ -1110,35 +1129,21 @@ an ESLint selector rather than a list. ### Wave 3 — Clear the ground -**Implementation in progress — 2026-10-18.** ADR-0110–0113, their linked -contracts and the nineteen-increment plan were approved on 2026-10-16. -[Steps 1–8 are independently accepted](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) -on `development`, with whole Step 8 reviewed at `c83c8575`. The phase register -owns each increment's source, tests, cumulative review rounds, security sittings -and platform/publication evidence, including historical NONCLEAN R1–R24 records. -This live page links that history instead of repeating its superseded pending states. - -The maintainer authorized the initial completed-Steps-1–8 publication checkpoint, -[PR #95](https://github.com/HodeTech/Relavium/pull/95). The -[previous corrective review completion](phases/phase-2.5.5-hardening-and-remediation.md#pr-95-corrective-review-completion--2026-10-18) -remains attributed to `ba14b1bd`; it does not certify later changes. The second -maintainer systematic report reviews `a5a5d45e` and opens further corrections. -The [second correction register](phases/phase-2.5.5-hardening-and-remediation.md#pr-95-second-systematic-review-corrections--2026-10-18) -is canonical for that work. The -[second corrective source-review completion](phases/phase-2.5.5-hardening-and-remediation.md#pr-95-second-corrective-source-review-completion--2026-10-18) -accepts corrections at `ec0f3e10` after two NEW complete cumulative clean rounds -and a NEW separate clean security sitting. Current source-bound PR/push CI, -coverage, SonarCloud, same-archive consumers and physical control artifacts remain -required publication criteria; PR #95 owns their live result. Source review does -not imply merge or whole-wave acceptance. The later -[native Windows admission preparation](phases/phase-2.5.5-hardening-and-remediation.md#pr-95-native-windows-admission-preparation--2026-10-18) -retains the failed advisory attempt and separate test-preparation review scope; -production runtime is unchanged. The later [supplemental fixture acceptance](phases/phase-2.5.5-hardening-and-remediation.md#pr-95-supplemental-native-fixture-source-acceptance--2026-10-18) -records both owned-preparation and native-concurrency review scopes, including -both failed Windows attempts. PR #95 owns final pushed-head publication results. -Steps 9–19 have **NOT STARTED** and whole-wave acceptance remains Step 19. Check PR #95 before later implementation commits, -which extend its review scope. No new decision, credential, billing or provider -capture gate is introduced. +**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Steps 9–19 NOT STARTED — 2026-10-19.** +ADR-0110–0113, their linked contracts and the nineteen-increment plan were approved on 2026-10-16. +[Steps 1–8 were independently accepted](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) +on `development`, with whole Step 8 reviewed at `c83c8575`, and published as the scoped checkpoint +[PR #95](https://github.com/HodeTech/Relavium/pull/95); its [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) is the canonical record of +what merged. The phase register owns each increment's source, tests, cumulative review rounds, security sittings and +platform/publication evidence, including historical NONCLEAN R1–R24 records, the +[previous corrective review completion](phases/phase-2.5.5-hardening-and-remediation.md#pr-95-corrective-review-completion--2026-10-18), +the [second correction register](phases/phase-2.5.5-hardening-and-remediation.md#pr-95-second-systematic-review-corrections--2026-10-18), +the [second corrective source-review completion](phases/phase-2.5.5-hardening-and-remediation.md#pr-95-second-corrective-source-review-completion--2026-10-18) +(accepted at `ec0f3e10`), the [native Windows admission preparation](phases/phase-2.5.5-hardening-and-remediation.md#pr-95-native-windows-admission-preparation--2026-10-18) +and the [supplemental fixture acceptance](phases/phase-2.5.5-hardening-and-remediation.md#pr-95-supplemental-native-fixture-source-acceptance--2026-10-18). +Those scopes are not widened: the delta from `ec0f3e10` to the merged head is two independently accepted test-fixture +increments, documentation and the independent review's corrections at `a3e164b5`, which have author verification +only. Whole-wave acceptance remains Step 19. No new decision, credential, billing or provider capture gate is introduced. Decompose, guard and normalize every hot file a 2.6 workstream is about to land on. **The multi-file queue is Step 1 and runs first.** The approved diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index 436ff9b0..f97f8739 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -2,7 +2,7 @@ > Status: Living -> Last updated: 2026-10-18 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are accepted on `development`, with PR #95 under review. Existing approved deferrals remain. +> Last updated: 2026-10-19 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Steps 9–19 not started). The independent PR #95 review's unactioned findings are recorded below. Existing approved deferrals remain. > The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) records completed prerequisites; full M/N lifecycle and other scheduled work remain with their owners. > > Earlier triage: the Phase-2.6 rewrite triaged every open item, the 2026-07-19 full-project @@ -368,6 +368,75 @@ Severity is the review's verified rating. Check an item off in the PR that resol there is concrete surface demand or telemetry showing operators need an earlier signal. *(1.AC; ADR-0028; config-spec.md; workflow-yaml-spec.md)* +## PR #95 independent review — unactioned findings (2026-10-18) + +An independent systematic review of [PR #95](https://github.com/HodeTech/Relavium/pull/95) at `19c99968` +reported no blocker or high finding. Its medium and three of its four low findings were corrected in `a3e164b5` +([disposition table](phases/phase-2.5.5-hardening-and-remediation.md#independent-review-of-the-published-head-and-its-corrections)); +what follows was **not** actioned. These are `- [ ]` (unscheduled, actionable) items, not approved deferrals: +no maintainer ruling is recorded for them. Items marked *(reviewer)* are quoted +from that review and were not re-verified by the author; items marked *(author)* were measured or counted here. + +- **DONE 2026-10-19 — SonarCloud's quality gate was red on `main` after every merge.** *(author)* The `main` + analysis failed `new_security_rating` (C, required A) against the `previous_version` baseline (2026-06-04) because nine + vulnerabilities were open there: seven `githubactions:S6505` ("omitting `--ignore-scripts`") on the + `pnpm install --frozen-lockfile` lines in `.github/workflows/ci.yml` (120, 432, 463, 487), `release.yml` (94) and + `models-catalog.yml` (51, 95); one `jssecurity:S8707` at `tools/budget-replay-compat/lifecycle-smoke.mjs:47`; one + `javascript:S4036` at `tools/cli-package/aggregate.mjs:58`. The gate had failed after the PR #90, #92, #94 and #95 + merges while the PR-level analyses passed, because SonarCloud statuses are per branch and the existing dispositions lived + on PR branches and on other `main` records. SonarCloud is not a required check. At the maintainer's direction the + dispositions were applied on `main` through the SonarCloud API after re-verifying each rationale at `6d2a7372`: the seven + S6505 and the S8707 as false positives (same reasons as their `main` siblings; the install-script posture comment at the top of + `ci.yml` and the Sonar backlog section below), the S4036 accepted as on the PR #95 branch. SonarCloud then reported zero + open vulnerabilities and an OK gate for `main`. No rule, threshold or gate was changed; the per-issue record is local + and Git-ignored. *(was medium · process)* +- [ ] **Native-lifetime watchdogs are fixed 5 s bounds inside 10 s cases (`LOW-1`).** *Low.* The watchdogs in + `apps/cli/src/process/native-process-owner.test.ts`, `engine/tool-host/os-native-lifetime.test.ts`, + `render/editor-native-lifetime.test.ts`, `render/fixtures/native-editor.mjs` and + `tools/cli-package/smoke/native-process.test.mjs` assert that a slow-but-correct close did not trip them, so a + loaded runner produces red without a defect. *(author)* The constants are **coupled** — the Home editor case + has a 5 s fixture watchdog, a 3 s ready wait, a 6 s release wait and a 10 s case bound — so raising one alone + does nothing, and three runs of the three vitest files (52 tests) passed at a load average of 80–96 on macOS. + The reviewer could not observe the Windows root cause either. **Trigger: a Windows trace that names the bound + that fired**; retune the coupled set together then. *(low · the files above)* +- [ ] **`bounds admitted-size failed header and URL preparation` is load-sensitive.** *Low.* *(author)* It allows + 5 s for two 8 MiB scrubs that take about 0.75 s in total, and it failed twice with the load average near 65 (6.4 s + in the run that logged the duration). Measured alternately, the pre-change code and the corrected code take the same + time, so the failures were load, not the correction. The test + guards against a quadratic regression, so any loosening must stay far below minutes. **Trigger: the next CI + failure of this case.** *(low · packages/llm/src/scrub-secrets.test.ts, `SECRETS_B2_HEADER_PREPARATION`)* +- [ ] **Raw exit-code literals where `EXIT_CODES` names exist.** *Nit.* *(reviewer)* reported + `result === 5 || result === 7 || result === 8` in `apps/cli/src/chat/session-surface-lifetime.ts:54`; *(author)* + the same literals occur in four places: that site, `commands/workflow-lifetime.ts:178`, + `process/command-lifetime.ts:110` and `commands/drive.ts:225`. Behaviour-neutral, but it sits on the exit-code + precedence that the review's red-team matrix held, so rerun the precedence tests with the change. + *(nit · the four files above)* +- [ ] **CLI lifetime and API-drift nits.** *Nit, (reviewer).* `chat/session-ink-input.ts:40-42,124-131` guards only + the `readable` listener (Ink's kitty `data` path is latent because `kittyKeyboard` is unused); + `home/drive-home.tsx:165-166,1067,1115-1116` keeps `HomeDeps.exit` although the owned path never calls it; + `commands/lazy-dispatch.ts:49` imports the media route without the cause-wrap the sibling routes carry. + *(nit · the files above)* +- [ ] **Test-evidence nits.** *Nit, (reviewer).* `process/pure-substrate-graph.test.ts:117-133` — the purity-guard + root set omits the ADR-0111 eager-boot files (the reviewer verified that opaque marking fixes it); three + no-argument `toHaveBeenCalled()` positives outside `native-process-owner.test.ts`; + `packages/llm/src/adapters/list-models-lifetime.test.ts:122` labels a fake-`fetch` case "real installed SDK"; + `apps/cli/src/harness/concurrency.e2e.test.ts:434-441` uses a fixed 100 ms contention margin; + `tools/cli-package/consumer-lifetime.test.mjs:54-59` pre-marks every synthetic case `passed: true` (by design; + real detection is the packed-consumer CI). *(nit · the files above)* +- [ ] **Tooling and documentation nits.** *Nit, (reviewer).* Unreachable `TAR_FORMAT` branches at + `tools/cli-package/archive.mjs:126,135`; no `break` in the ancestor loop at `tools/cli-package/smoke/worker.mjs:64-65` + (the outermost `node_modules` wins); and repeated heading texts in the phase register, whose later occurrences + take a `-1` slug suffix. *(author)* A scan finds repeated H2–H4 headings (the 2.5.5.B–I sub-stream headings and the + Wave 2 checkpoint's sub-headings among them); no link is broken today. *(nit · the files above)* +- [ ] **Informational items to keep in view.** *(reviewer)* `packages/mcp/src/manager.ts:228-236` — a comment omits + the bare `Error` in the throwing-observer branch (behaviour is canonical); `packages/core/src/tools/bounding.ts` — + the removed 4k pre-clip in `makeSummary` is required for correctness (8 MiB takes about 49 ms, about 200 ms with + line feeds) and the alias DAG is `2^d` (depth 18 is 786k nodes in 404 ms) for host/embedder graphs only, never model + JSON; `packages/core/src/tools/builtins.ts:451-452` — the `git_commit` description promises human-gate approval + while no producer sets `gateApproved=true` (not verified by the reviewer); `commands/workflow-lifetime.ts:204` + can return a raw signal code above 9 that the physical exit never uses, a note for future embedders. No action + is proposed for any of these. + ## Phase 2.6.5 deferrals — `CR-93` and `CR-95` (long half), recorded 2026-09-06; `CR-70`, `CR-94` (remainder) and the in-turn overflow continuation, recorded 2026-09-14; the errored-or-aborted turn residual, the pre-upgrade free-page bytes and the estimator's calibration, recorded 2026-09-18 **Phase closure — 2026-10-11:** W8 merged to `main` in PR #92 and Phase 2.6.5 is complete diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 91f78042..ac787db1 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -3672,6 +3672,10 @@ No credential values or request captures belong in tracked evidence. | 8c | **Recorded split — workflow and render integration:** run/gate/budget/driver owners, real return/throw health priority, cancellation, terminal/job-control leases and force-quit routing. No timer certifies DB close or receipt settlement. | | 8d | **Recorded split — sessions, terminal and activation:** fresh/resumed/one-shot/Home/pending/replaced session lifetimes, retained publication failure and shared actual cleanup promises; connect caught slash/rebuild faults, restore terminal custody before diagnosis, then activate early coordinator/background/sink integration. Actual native/platform/packed controls and complete cumulative Step 8 acceptance follow. | | 9 | **Release map/custody substrate:** #46/ADR-0111, independently reviewed after the finite graph. Upstream TS maps, sole prepack final build, pre-hash trailer removal/staging, exact tarball/CLI+engine symbolication, encrypt-once candidate with permissioned prebuild discovery/reservation → pack/reuse → smoke → archive → npm → finalization. Required artifact/platform aggregation, all rerun modes, lost-staging refusal and permission-enforcing offline API controls. Dedicated archive/crypto security sitting. Fixture proof never claims production recipient/custody configuration. | +| 9a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage #46/ADR-0111 unchanged, no new contract) — release maps, final build and symbolication:** fresh uncached upstream original-TypeScript maps, the sole prepack final CLI build, final trailer removal before hashes, map and private-graph staging outside public artifacts and caches, exact tarball-to-final-JS binding, and real packed CLI and engine frame symbolication with wrong JS/map/source and missing-map negative controls. Ordinary builds remain map-free. | +| 9b | **Recorded split — recipient custody:** synthetic-only isolated GnuPG controls (public-only home/keyring, pinned primary and encryption-subkey fingerprints, machine-readable refusal of missing, expired, revoked or unusable recipients), encrypt once and reuse the original ciphertext. Explicit crypto/custody security sitting. Fixture proof never claims production recipient or custody configuration. | +| 9c | **Recorded split — offline release state controller:** permission-enforcing offline GitHub/npm controller, permissioned prebuild discovery and reservation, and every rerun, partial, conflict and expiry state, with no overwrite, rebuild or re-encryption on uncertain history. | +| 9d | **Recorded split — workflow, runbook and whole-Step-9 landing:** least-privilege actual workflow dependency graph, required artifact/platform aggregation and exact-artifact consumers, lost-staging refusal and the release runbook update. The whole-Step-9 cumulative review retains all 9a–9d changes with preservation traces into Step 8, and the dedicated archive/crypto security sitting closes the step. | | 10 | **CLI behaviour/discovery integration:** #45/#264/#265/#268/#13/#17/#18/#310/#309/#19 under ADR-0110, consuming Steps 6–9. Production owner category matrix, real root/subtree JSON help, pricing/chat aliases, non-echoing Commander refusal and signal-safe deletion consent. G15 is refuted and receives real engine/store/log preservation evidence, with no invented token coalescing. Repeat affected packed gates. | | 11 | **Chat normalization and abort:** #236/#237/#242/#307 → #218 → #21/#23. Canonical token/money source with approved additive LLM export, preserved two-decimal cap formatting and real immediate abort while a turn is blocked. Audit Home's imports. | | 12 | **Chat decomposition:** #220/#244 after preceding chat fixes. Split lifecycle/drivers, line handler, printers and reseat/clear ownership. Preserve compatibility entry points, W7/W8 receipts, unknown-usage, full-screen bounds, disclosure activation and cancellation. | @@ -9141,3 +9145,132 @@ criteria; [PR #95](https://github.com/HodeTech/Relavium/pull/95) owns their live result. Source acceptance does not assert a green retry or completed publication. Steps **9–19 have NOT STARTED**. No merge, release, new user decision or deferral is claimed. + +### Wave 3 Steps 1–8 merge checkpoint — 2026-10-18 + +[PR #95](https://github.com/HodeTech/Relavium/pull/95) merged to `main` at +**22:44:24 UTC (01:44:24 Europe/Istanbul, 2026-10-19)**, merge commit +`6d2a737231e2445c6d9d54b2d7449084e036ade4` (parents `ab08209e` and `a3e164b5`). The merged +head is `a3e164b5608d386fc3db7fa3b8483d0adf2ed0fd`; the merge tree +(`415af9662f5690b675ebaf4ccfff5840ea6e3f29`) is byte-identical to it. GitHub records +649 changed files, +81,844 / −7,343. **Wave 3 Steps 1–8 are complete on `main`. Wave 3 is +not: Steps 9–19 have NOT STARTED and whole-wave acceptance remains Step 19.** ADR-0110–0113 +remain Accepted. The PR was the maintainer-authorized, scoped publication checkpoint recorded +in [Wave 3 Steps 1–8 scoped publication](#wave-3-steps-18-scoped-publication--2026-10-17); it +claimed no whole-wave acceptance and this checkpoint does not add one. + +#### Completed Steps 1–8 scope + +| Step | Complete on `main` | +| --- | --- | +| 1 | Shared close handling, safe source labels with UTF-8 admission parity, exhaustive session/picker/run switches and actual reasoning rendering. | +| 2 | Shared admission and diagnostics: status/duplicate matrices, shared parser reason helper and config numeric bounds. | +| 3 | Secret provenance at public sinks (ADR-0113): engine-owned context/input provenance across fresh, resume, condition, transform and merge paths. | +| 4 | Typed error/envelope substrate (`RelaviumError`, migrated classes, cross-bundle structural guards). | +| 5 | Tools and engine tails: bounds before effects, model-visible semantics, ordinal helper, path-local cycle guards, cancellation race and deterministic plan equivalence. | +| 6 | Pure process/discovery substrate (ADR-0110): categories, early flag grammar, exit/site inventory, descriptor registry and invocation owner registry. | +| 7 | Lazy runtime, finite compiled graph, exact-tarball inspection and the required Node 22.0.0 plus Linux/macOS/Windows consumers (ADR-0111). | +| 8 | Creation-owned native process integration, kind-specific owners, terminal custody and the total fatal budget across workflow, render and session lifetimes. | + +#### Review attribution + +Each step keeps its own acceptance register; none is widened by the merge. Whole Step 8 +is accepted at `c83c8575` (two NEW complete cumulative CLEAN rounds, 342 paths / 1,842 +ordered hunks, separate clean security sitting). The two PR #95 correction cycles are +accepted at `ec0f3e10` ([second corrective source-review completion](#pr-95-second-corrective-source-review-completion--2026-10-18)), +and the two later native test-fixture increments (`154d832d`, `1c7aec25`) have their own +[supplemental acceptance](#pr-95-supplemental-native-fixture-source-acceptance--2026-10-18). +Detailed review records remain local and Git-ignored. + +The difference between `ec0f3e10` and the merged head is exactly twelve non-documentation +paths plus three documentation files. Two test files are the independently accepted fixture +increments above. The other ten are the commit below, which has **no independent acceptance**. + +#### Independent review of the published head and its corrections + +After the accepted sources, a further independent systematic review of the PR at `19c99968` +(reviewer-reported: two rounds, 14 agent sessions in total, all 648 changed files read at diff level, +about 5,000 targeted test runs, five red-team attacks on the headline invariants) reported **no blocker, no high, one +medium, four low and about seventeen nit/info/suggestion items**, with four items withdrawn after +its own verification. The attacks did not break a headline claim; the test-evidence attack held +only partially (LOW-4). Its detailed records stay local. The author reproduced each finding +before acting. Corrections landed as `a3e164b5`: + +| Finding | Disposition | +| --- | --- | +| **MEDIUM-1** — URL userinfo not masked when its scheme is glued to a word character (`xhttps://user:pass@host`) | Reproduced at `19c99968` through both `scrubSecrets` and the CLI sanitizer. Fixed in `packages/llm/src/scrub-secrets.ts`: the rule returns to its pre-PR word-adjacency (no `\b`) and supplies its own joined pattern, because the split-credential scanner otherwise strips two characters from a pattern that has none. Fourteen new `scrub-secrets` cases and one CLI case; reverting the production change fails all fourteen (the CLI case failed against the stale build before it was rebuilt). | +| **LOW-4** — failed-spawn test does not pin the native close join | Confirmed by mutation: removing `reservation.whenClosed` left all three `process-close` tests green. The reviewer's suggested `activeOwners === 0` assertion was also tried and **did not** kill that mutant, because the owner retires synchronously inside its own `close` handler on POSIX. Replaced by a case that gates the reservation's `whenClosed` and observes publication; it fails only when the join is removed. | +| **LOW-2** — default 1 s `vi.waitFor` on a real spawned child | Fixed: the four waits in `native-forwarding.test-support.ts` carry an explicit 4 s bound, below the 5 s default case timeout. | +| **LOW-3** — POSIX `ps` suite not skipped on Windows | Fixed: `describe.skipIf(process.platform === 'win32')`. Prettier re-indented the suite body, so the raw diff of that file is 215 lines; `git show -w` shows the 7-insertion / 2-deletion semantic change. | +| **LOW-1** — fixed 5 s native watchdogs | **Not changed.** Three runs of the three affected native files (52 tests) all passed at a load average of 80–96, the reviewer could not observe the Windows root cause, and the constants are coupled to neighbouring bounds, so a retune would be speculative. Recorded in [deferred-tasks.md](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18). | +| Nit — import above the module docblock in `parser.ts`, `agent-parser.ts` | Fixed (no behaviour change). | +| Nit — `packages/shared` curated export list | Closed without change: it is an illustrative existence sample, not an exhaustive contract. | +| Other nits and info items | Open, in the same deferred-tasks section. | + +The production change restores a pre-PR behavior, but it touches a secret-scrubbing rule and +the repository's causal controls: the `diagnostic-raw-password-userinfo` source anchor moved to +the new `URL_USERINFO_PATTERN` constant, and the pinned complete workloads changed (89→103, +179→180, 277→278) because the files gained cases. **Verification is the author's only.** The +full local gate was run in two parts: `pnpm run ci` through the workspace tests, builds, format, +fences, bundle closure and smokes; then, after the anchor/pin update, `test:acceptance-tools` +(633 of 633) and the secrets, wire, usage and filesystem mutation gates (exit code 0 each, the +userinfo control 103 / 98 with five failing cases / 103), plus tools lint, typecheck and format and +the affected suites under `CI=true GITHUB_ACTIONS=true`. A second complete `pnpm run ci` after +the pin update and a coverage run were not performed. No independent review rounds and no security +sitting cover `a3e164b5`; none is claimed. + +#### Publication state observed at merge + +The merged head's [PR CI](https://github.com/HodeTech/Relavium/actions/runs/39169681878) and +[push CI](https://github.com/HodeTech/Relavium/actions/runs/39169678515) started at 21:51 UTC. +Observed at 22:48 UTC, four minutes after the merge, each had 14 of 15 jobs completed +successfully (including the four same-archive installed-CLI consumers and the causal-control +families) and no failed job. The aggregate `lint · typecheck · test` job, one of the two +required status checks on `main` (with `engine coverage floor (llm, mcp)`), was still running; in the PR run it +started at 22:46:21 UTC, after the merge. The merge commit's own +[push CI on `main`](https://github.com/HodeTech/Relavium/actions/runs/39173540595) started at +22:44:30 UTC. **That observation asserted none of the eventual conclusions.** Earlier full-CI, +coverage, SonarCloud and platform results stay attributed to the sources recorded in the preceding +registers and are not carried to `a3e164b5` or the merge commit. + +**Later observation, 2026-10-19 06:53 UTC.** All three runs completed with conclusion `success`, +**15 of 15 jobs each**, and both required checks succeeded on the merged head (`lint · typecheck · test` +completed 22:54 UTC in the push run and 23:00 UTC in the PR run). Job logs, coverage figures and +test counts were not re-read for this record. SonarCloud's PR analysis passed its quality gate +(102 new issues, 86 accepted). Its analysis of the **`main` merge commit failed the quality gate**: +`new_security_rating` C against the required A, from 9 open vulnerabilities on the `previous_version` +baseline (2026-06-04) — seven `githubactions:S6505` findings on workflow `pnpm install` lines +(created June–July, deliberately declined with a recorded rationale but still open on the `main` +branch, where SonarCloud already carries five other S6505 findings as false positives), one +`jssecurity:S8707` (created 2026-10-08; seventeen sibling findings are already false positives +on `main`) and one `javascript:S4036` (created 2026-10-17; won't-fix on the PR #95 branch, open on +`main`). SonarCloud statuses are per branch, so these are existing dispositions not yet applied +on `main`. The same gate also failed on the `main` merge +commits of PR #90, #92 and #94, so this was a standing condition of `main`, not a regression of +PR #95, and SonarCloud is not a required check. **Resolved 2026-10-19 10:50 UTC:** at the +maintainer's direction the author applied those dispositions on `main` through the SonarCloud +API, each with a reviewed comment (the seven S6505 and the S8707 as false positives, the S4036 +accepted), after re-verifying the recorded rationale against the repository at `6d2a7372` — the +pinned pnpm 9.12.3, the two-package `onlyBuiltDependencies` allowlist, no workspace install hooks, +the S8707 path coming from a fresh owned scratch directory, and `RELAVIUM_PACKAGE_SOURCE_SHA` set by +both workflow steps that run the aggregator. SonarCloud then reported **zero open vulnerabilities +and an OK quality gate for `main`**. No rule, threshold or quality gate was changed. The failed +check-run attached to the merge commit itself is historical and unchanged. Recorded in +[deferred-tasks.md](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18). + +#### Exit status and Step 9 as next work + +**Wave 3 Steps 1–8 are on `main`; Wave 3 remains in progress.** Open after this merge: Steps 9–19 +and whole-wave acceptance (Step 19: A/E/G and M2.5.5-3 evidence, promoting core's ≥90% line/branch +floor to required CI, the reconciled EXIT:4 reading, two new whole-wave cumulative reviews); +an independent review of the `a3e164b5` delta; and the unactioned review findings. Phase 2.5.5 exit criteria 4–8, sub-stream I, later +product waves and the existing approved deferrals keep their canonical homes. The merge adds no +deferral, ADR or release. + +**Next is Step 9 — release map/custody substrate** (#46, ADR-0111): upstream TypeScript maps, a +sole prepack final build, exact tarball symbolication, the encrypt-once candidate through +discovery/reservation → pack/reuse → smoke → archive → npm → finalization, with a dedicated +archive/crypto security sitting. The maintainer-approved 9a–9d subdivision is recorded in the +[increment table](#proposed-implementation-increments), and 9a is first. Steps 10–19 follow the +approved plan in order. New implementation +commits on `development` now start a new publication checkpoint; PR #95 no longer bounds their review scope. From c64f3c39efc9154068ad9cca88d213944f5c8198 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 19 Oct 2026 15:36:48 +0300 Subject: [PATCH 02/97] fix(llm): examine glued url schemes in the split-credential scan An independent security sitting on a3e164b5 found that a scheme glued to a word character (xhttps://...), split by LF/TAB and carrying a raw @ in the credential, left a partial mask with part of the credential visible. The split-credential scanner still skipped any start preceded by a word character, although the URL rule no longer has a word boundary. The URL rule's boundary now blocks no start, and the word-character shortcut applies only when a rule's boundary rejects word characters. The anchored scanner gate and the URL pattern line are unchanged, so no causal-control anchor moves. - tests: two regression cases (reverting the gate fails exactly those), plus an underscore glue, a split inside the scheme, a multi-URL case and three characterisation cases for the fail-closed trade-offs the review found (prose before a URL and a later @ redacts the whole diagnostic; a longer scheme name such as sftp matches by suffix). A test comment wrongly said ws/wss/ftp/ftps were masked before the PR; they are the PR's additions - comments: the stale boundary and slice(2) contract text is corrected - pins: the scrub-secrets file set 103 -> 111, in mutations.mjs and provenance.test.mjs; the secrets-controls README counts follow (278, 111, 180, 1,343), measured from the control run Co-Authored-By: Claude Sonnet 5.5 --- apps/cli/src/render/sanitize.test.ts | 3 ++ packages/llm/src/scrub-secrets.test.ts | 46 ++++++++++++++++++++++- packages/llm/src/scrub-secrets.ts | 39 +++++++++++++------ tools/causal-controls/provenance.test.mjs | 2 +- tools/secrets-controls/README.md | 10 ++--- tools/secrets-controls/mutations.mjs | 2 +- 6 files changed, 81 insertions(+), 21 deletions(-) diff --git a/apps/cli/src/render/sanitize.test.ts b/apps/cli/src/render/sanitize.test.ts index 00a18161..6a96c9b5 100644 --- a/apps/cli/src/render/sanitize.test.ts +++ b/apps/cli/src/render/sanitize.test.ts @@ -259,6 +259,9 @@ describe('sanitizeUntrusted — the failure paths carry arbitrary text (#W15-8)' for (const separator of ['\n', '\t']) { expect(sanitize(`xhttps://us${separator}er:opaque-value@host/x`)).toBe('[REDACTED]'); } + // Glued scheme + split credential + a raw `@` in the credential: no partial mask may remain. + expect(sanitize('xhttps://user:p@ss\tword@host/x')).toBe('[REDACTED]'); + expect(sanitize('xhttps://u@e.com:pw\nxyz@host/x')).toBe('[REDACTED]'); } }); }); diff --git a/packages/llm/src/scrub-secrets.test.ts b/packages/llm/src/scrub-secrets.test.ts index 31aac2bb..fb9c9121 100644 --- a/packages/llm/src/scrub-secrets.test.ts +++ b/packages/llm/src/scrub-secrets.test.ts @@ -83,8 +83,10 @@ describe('finite diagnostic credential backstop regressions (PR #95 M2)', () => expected: `connect ${scheme.toUpperCase()}://[REDACTED]@provider.invalid/v1 failed`, }, ]), - // A scheme glued to a preceding word character was masked before this PR; a `\b` on the rule - // let it through (PR #95 review, MEDIUM-1). Adjacent and LF/TAB-split forms are both pinned. + // A scheme glued to a preceding word character: the pre-PR matcher masked it for http(s), and a `\b` on + // the rule let it through (PR #95 review, MEDIUM-1). ws/wss/ftp/ftps are this PR's additions and get + // the same word-adjacency, so those cases pin new behaviour, not parity. Adjacent and LF/TAB-split + // forms are both pinned. ...['http', 'https', 'ws', 'wss', 'ftp', 'ftps'].flatMap((scheme) => [ { input: `x${scheme}://alice:opaque-password@provider.invalid/v1`, @@ -99,6 +101,29 @@ describe('finite diagnostic credential backstop regressions (PR #95 M2)', () => input: `xhttps://ali${separator}ce:opaque-password@provider.invalid/v1`, expected: '[REDACTED]', })), + // A glued, split credential that also contains a raw `@` used to leave a partial mask with the rest + // of the credential visible: the scanner skipped starts preceded by a word character (security + // sitting, PR #95 post-merge review). A clean-prefix twin was already redacted whole. + { + input: 'x\u0000https://u@e.com:pw\nxyz@provider.invalid/v1', + expected: '[REDACTED]', + }, + { + input: 'xhttps://user:p@ss\tword@provider.invalid/v1', + expected: '[REDACTED]', + }, + { + input: '_https://alice:opaque-password@provider.invalid/v1', + expected: '_https://[REDACTED]@provider.invalid/v1', + }, + { + input: 'xht\ntps://alice:opaque-password@provider.invalid/v1', + expected: '[REDACTED]', + }, + { + input: 'a http://u1:p1@h1.invalid/x and xhttps://u2:p2@h2.invalid/y', + expected: 'a http://[REDACTED]@h1.invalid/x and xhttps://[REDACTED]@h2.invalid/y', + }, ...['api-key', 'x-api-key', 'x-goog-api-key'].flatMap((name) => [ { input: `header ${name}: opaque-header-Z29 rejected`, @@ -145,6 +170,23 @@ describe('finite diagnostic credential backstop regressions (PR #95 M2)', () => expect(error.code).toBe(expected); }); + // Documents, and does not endorse, two consequences of a URL rule with no word boundary. Both fail + // closed: they blank or widen a diagnostic, they never reveal one. The whitespace-stripped residual probe + // glues the prose before a URL to its scheme, and a longer scheme name matches by its suffix. + it.each([ + { + input: 'Error: ECONNREFUSED http://127.0.0.1:11434 contact admin@corp', + expected: '[REDACTED]', + }, + { + input: 'MCP server at ws://127.0.0.1:8080 refused; email support@vendor.com', + expected: '[REDACTED]', + }, + { input: 'sftp://scm@code.example/x', expected: 'sftp://[REDACTED]@code.example/x' }, + ])('fails closed on the documented trade-off $input', ({ input, expected }) => { + expect(scrubSecrets(input)).toBe(expected); + }); + it('protects each internal LF/TAB split position in the finite Bearer/Basic labels', () => { for (const label of ['Bearer', 'Basic']) { for (const separator of ['\n', '\t', '\n\t']) { diff --git a/packages/llm/src/scrub-secrets.ts b/packages/llm/src/scrub-secrets.ts index 9d9921a3..b547d979 100644 --- a/packages/llm/src/scrub-secrets.ts +++ b/packages/llm/src/scrub-secrets.ts @@ -8,7 +8,8 @@ import { stripTerminalControls } from '@relavium/shared/terminal-safe'; * names, URL userinfo and finite API-key/Bearer/Basic headers. Each non-query rule probes the original diagnostic * independently, including terminal-normalized and LF/TAB-joined views, before any * other rule can erase its identifying context. A match revealed by normalization masks the whole - * diagnostic. The finite patterns check boundaries in the original text and preserve already-masked spans; + * diagnostic. The finite patterns check boundaries in the original text (the URL rule has none, like the + * pre-PR matcher, and so fails closed on prose that sits next to a URL) and preserve already-masked spans; * this is not universal fragment discovery. Query names are excluded from the residual probe. * Pure and total for string input: regex replacements and a * finite residual check, with no parsing or external calls on the error path. @@ -27,12 +28,19 @@ interface SecretRule { } /** - * Deliberately NOT word-bounded: the pre-PR matcher masked `xhttps://user:pass@host`, and a `\b` here - * would let any scheme glued to a preceding word character through unmasked. The rule has no `\b` - * prefix, so it also supplies its own joined form rather than letting the scanner strip one. + * Deliberately NOT word-bounded: the pre-PR http(s) matcher masked `xhttps://user:pass@host`, and a `\b` + * here would let any scheme glued to a preceding word character through unmasked. This PR added + * ws/wss/ftp/ftps under the same rule. The rule has no `\b` prefix, so it supplies its own joined form + * (the split-credential scanner would otherwise strip two characters that are not there) and its + * `boundary` blocks no start. Consequences, all fail-closed and pinned by tests: a longer scheme name + * ending in one of these (`sftp://u@h`) matches by suffix, and the whitespace-stripped residual probe + * can turn prose followed by a URL and a later `@` into a whole-diagnostic `[REDACTED]`. */ const URL_USERINFO_PATTERN = /(?:https?|wss?|ftps?):\/\/[^/\s]*@/gi; +/** A start boundary that never matches: no preceding character makes a start ineligible. */ +const NO_BOUNDARY = /(?!)/; + /** One private authority per existing credential family, shared by probes and output masking. */ const NON_QUERY_SECRET_RULES: readonly SecretRule[] = [ // OpenAI sk-/sk-proj-/sk-svcacct- and Anthropic sk-ant- key prefixes. @@ -52,7 +60,7 @@ const NON_QUERY_SECRET_RULES: readonly SecretRule[] = [ pattern: URL_USERINFO_PATTERN, joinedPattern: URL_USERINFO_PATTERN, replacement: maskUrlUserinfo, - boundary: /\w/, + boundary: NO_BOUNDARY, }, // Exact literal header names, also in quoted diagnostic objects. No arbitrary header discovery. { @@ -108,7 +116,10 @@ function applyRule(text: string, rule: SecretRule): string { /** * Probe LF/TAB joins at eligible ORIGINAL starts before a greedy match can hide a later boundary. * All boundary-bearing rules use this scanner and their canonical matcher (or finite header-label - * override). Prefixes start with ASCII letters; that prefilter skips only impossible starts. + * override). A rule that has a `joinedPattern` uses it as written; any other rule MUST begin its pattern + * with `\b`, which is stripped here (deleting an explicit `joinedPattern` from a rule without `\b` + * throws on every LF/TAB input, which the suite catches at once). Prefixes start with ASCII letters; + * that prefilter skips only impossible starts. * Keys have one token run after a bounded prefix/minimum. Header labels are bounded and their * trailing whitespace cannot contain another label. URL authority excludes slash, so a later URL * prefix delimits a failed authority scan. Successful matches advance both cursors over their span; @@ -124,6 +135,8 @@ function hasSplitCredential(text: string, joined: string, rule: SecretRule): boo // Native scanning avoids a JS/coverage branch for every byte of unrelated large text. if (!new RegExp(source, canonical.flags).test(joined)) return false; const pattern = new RegExp(source, canonical.flags.replace('g', 'y')); + // Only a rule whose boundary rejects ASCII word characters may skip word-preceded starts. + const wordBlocksStart = rule.boundary.test('a'); let originalOffset = 0; let joinedOffset = 0; while (originalOffset < text.length) { @@ -132,17 +145,19 @@ function hasSplitCredential(text: string, joined: string, rule: SecretRule): boo originalOffset += 1; continue; } - // Both finite boundary classes reject ASCII word characters. Skip those impossible + // The key and header boundary classes reject ASCII word characters. Skip those impossible // starts without allocating/testing RegExp objects for every admitted-size character; - // the rule's own boundary and canonical matcher still decide every eligible start. + // the rule's own boundary and canonical matcher still decide every eligible start. The URL + // rule blocks no start, so a glued scheme (`xhttps://`) is examined like any other. const code = text.charCodeAt(originalOffset); const previous = text.charCodeAt(originalOffset - 1); const letter = (code >= 65 && code <= 90) || (code >= 97 && code <= 122); const previousWord = - (previous >= 48 && previous <= 57) || - (previous >= 65 && previous <= 90) || - (previous >= 97 && previous <= 122) || - previous === 95; + wordBlocksStart && + ((previous >= 48 && previous <= 57) || + (previous >= 65 && previous <= 90) || + (previous >= 97 && previous <= 122) || + previous === 95); if (letter && !previousWord && !rule.boundary.test(text[originalOffset - 1] ?? '')) { pattern.lastIndex = joinedOffset; const match = pattern.exec(joined); diff --git a/tools/causal-controls/provenance.test.mjs b/tools/causal-controls/provenance.test.mjs index d1f19400..8b553a68 100644 --- a/tools/causal-controls/provenance.test.mjs +++ b/tools/causal-controls/provenance.test.mjs @@ -317,7 +317,7 @@ test('secrets workload corrections retain independent complete-count and reader- 'diagnostic-raw-password-userinfo', 'diagnostic-joined-original-boundary-recovery', 'diagnostic-header-original-boundary-recovery', - ].map((id) => [id, ['packages/llm/src/scrub-secrets.test.ts'], 103]), + ].map((id) => [id, ['packages/llm/src/scrub-secrets.test.ts'], 111]), ...['diagnostic-inline-projection-order', 'diagnostic-original-control-boundary'].map((id) => [ id, ['apps/cli/src/render/sanitize.test.ts', 'apps/cli/src/process/render-error.test.ts'], diff --git a/tools/secrets-controls/README.md b/tools/secrets-controls/README.md index bce9595c..20f9e206 100644 --- a/tools/secrets-controls/README.md +++ b/tools/secrets-controls/README.md @@ -48,7 +48,7 @@ other replacement erases its evidence, followed by the finite query replacement. The views include the canonical terminal-normalized text and a whitespace-joined residual view. The split literal query name remains an independent CLI obligation. Across the thirteen Step 3 controls, 191 named assertion witnesses are required. The shared -overlap controls collect 277 tests across the LLM, CLI sanitizer and renderer files. +overlap controls collect 278 tests across the LLM, CLI sanitizer and renderer files. The B2 controls below pin the conservative LF/TAB continuation rule when a first fragment already meets a key-length floor. Broader whitespace residual checks retain their finite scope; neither mechanism claims universal secret detection. @@ -178,8 +178,8 @@ check, split scheme labels, split API-header labels and raw `?`/`#` password rec independently. Four LLM controls remove the before-masking LF/TAB probe, its original left-boundary check, split API-header recognition and raw-password recognition. Every core phase executes the complete 142-case bounding file; every new LLM phase -executes the complete 89-case scrubber file. Independent ordinary tooling tests pin -these inventories, the 277-case overlap cohort, 179-case CLI diagnostic cohort and +executes the complete 111-case scrubber file. Independent ordinary tooling tests pin +these inventories, the 278-case overlap cohort, 180-case CLI diagnostic cohort and 18-case command-export cohort. Actual collection still has to match in every phase. The boundary controls preserve an embedded public-word key-shaped run containing an @@ -240,7 +240,7 @@ The independent `diagnostic-original-control-boundary` removal strips terminal controls before the initial shared scrubber can observe their original credential boundary. Eight multiline/inline sanitizer cases cover removed C0, DEL, C1 and bidi units; the same four real renderer cases must fail their distinct -`SECRETS_B2_DISPLAY_BOUNDARY` assertion. Both phases execute the complete 179-case +`SECRETS_B2_DISPLAY_BOUNDARY` assertion. Both phases execute the complete 180-case sanitizer/renderer cohort. The earlier CLI residual-query control remains independent and retains its required witnesses. Its removal now skips the actual remaining-span loop: the new outer raw scrubber can already mask ordinary matches, @@ -257,7 +257,7 @@ previous-word prefilter and the rule's own boundary predicate; retaining either would leave the tested policy active. Header/key recovery removals still change their original-rule boundary only, and all removals retain the same canonical matchers and bounded raw/joined standalone visitors. Current collection is 25 -complete files / 1,320 cases, with each control's exact workload and named target +complete files / 1,343 cases, with each control's exact workload and named target checked separately. These are updated declarations and author checks, not new physical causal captures or a clean review round. R3 is preserved as NONCLEAN; later corrective acceptance belongs to the owning phase register. Prior selected diff --git a/tools/secrets-controls/mutations.mjs b/tools/secrets-controls/mutations.mjs index 96d47e90..81f90dad 100644 --- a/tools/secrets-controls/mutations.mjs +++ b/tools/secrets-controls/mutations.mjs @@ -234,7 +234,7 @@ function diagnosticControl(id, before, after, targets, marker = diagnosticMarker before, after, ['packages/llm/src/scrub-secrets.test.ts'], - 103, + 111, targets[0], marker, targets.slice(1).map((target) => ({ target, marker })), From f597977a01f49fabed84e9922d6f5c77e1a34724 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 19 Oct 2026 15:36:48 +0300 Subject: [PATCH 03/97] test(cli): pin the native close join on every publication branch The new gated-reservation case pinned the join of the owner's whenClosed only for the failed-spawn publication. A mutant that kept the join on that branch alone and dropped it for natural exit, abort and timeout passed every test in tool-host/process*.test.ts. - process-close.test.ts: one table over four branches. Removing the join fails all four; keeping it only on the failed-spawn path fails natural exit, abort and timeout; dropping it only there fails failed-spawn. The closed-child wait now carries an explicit bound - native-forwarding.test-support.ts: the longer waitFor bound applies only to the one wait that observes a real child booting (3 s, leaving 2 s of the 5 s default case timeout). Three chained 4 s waits could exceed the case timeout and lose the named assertion, and the other waits only observe registry reactions, so they return to the default. The docblock now says so Co-Authored-By: Claude Sonnet 5.5 --- .../native-forwarding.test-support.ts | 16 +-- .../engine/tool-host/process-close.test.ts | 116 +++++++++++++----- 2 files changed, 90 insertions(+), 42 deletions(-) diff --git a/apps/cli/src/commands/native-forwarding.test-support.ts b/apps/cli/src/commands/native-forwarding.test-support.ts index e80f460f..3791569b 100644 --- a/apps/cli/src/commands/native-forwarding.test-support.ts +++ b/apps/cli/src/commands/native-forwarding.test-support.ts @@ -11,11 +11,13 @@ import { } from '../process/native-process-owner.js'; /** - * These waits observe a REAL spawned child (or the retirement reaction of one). vitest's default - * `waitFor` timeout is 1 s, which a cold or loaded worker can exceed before a child has booted. - * Kept under the 5 s default test timeout so a genuine failure surfaces as its own assertion. + * Applied to the ONE wait that observes a real child booting (its marker file): vitest's default + * `waitFor` timeout is 1 s, which a cold or loaded worker can exceed before a node child has started. + * It leaves 2 s of the 5 s default case timeout for setup; a failure after more than that is reported + * by the case timeout instead of the named assertion. The other waits here only observe registry + * reactions of an already-started or already-closed child and keep the default. */ -const REAL_CHILD_WAIT_MS = 4_000; +const REAL_CHILD_BOOT_WAIT_MS = 3_000; /** A physical child witness for host-only forwarding, with an independent tool timeout. */ export async function witnessNativeToolProcess( @@ -38,7 +40,6 @@ export async function witnessNativeToolProcess( // queued. Require its actual removal before measuring the stable native-parent baseline. await vi.waitFor(() => expect(invocation.registry.activeOwners).toBe(expectedParentOwners), { interval: 1, - timeout: REAL_CHILD_WAIT_MS, }); const pending = host.process.spawn( process.execPath, @@ -51,12 +52,11 @@ export async function witnessNativeToolProcess( expect(invocation.registry.activeOwners, 'NATIVE_PROCESS_ASSEMBLY_OWNERSHIP').toBe( expectedParentOwners + 1, ), - { interval: 1, timeout: REAL_CHILD_WAIT_MS }, + { interval: 1 }, ); expect(await pending).toMatchObject({ exitCode: 0, stdout: 'native-witness' }); await vi.waitFor(() => expect(invocation.registry.activeOwners).toBe(expectedParentOwners), { interval: 1, - timeout: REAL_CHILD_WAIT_MS, }); } @@ -103,7 +103,7 @@ export async function holdNativeToolProcess( expect(existsSync(marker)).toBe(true); // physical native entry, not only a reserved field expect(invocation.registry.activeOwners).toBe(before + 1); }, - { interval: 1, timeout: REAL_CHILD_WAIT_MS }, + { interval: 1, timeout: REAL_CHILD_BOOT_WAIT_MS }, ); return { close }; } catch (error) { diff --git a/apps/cli/src/engine/tool-host/process-close.test.ts b/apps/cli/src/engine/tool-host/process-close.test.ts index 140d7510..e126c815 100644 --- a/apps/cli/src/engine/tool-host/process-close.test.ts +++ b/apps/cli/src/engine/tool-host/process-close.test.ts @@ -127,39 +127,87 @@ describe('native child resource lifetime after an error (ADR-0103)', () => { expect(closed).toBe(true); }); - it('withholds the failed-spawn result until the owner reservation joins close', async () => { - // The child's own `close` event cannot pin this: the capability's handler only runs after it, so a - // rejection that skipped `reservation.whenClosed` would still see a closed child (and, on POSIX with no - // helper, an already-retired owner). Gate the reservation's join itself and observe the publication. - const real = nativeProcessesForInvocation(createCliInvocation()); - if (real === undefined) throw new Error('explicit invocation missing'); - const gate = deferred(); - const gated: NativeProcessController = { - ...real, - reserve: async (kind) => ({ ...(await real.reserve(kind)), whenClosed: gate.promise }), - }; - let closed = false; - vi.mocked(spawn).mockImplementationOnce((command, args, options) => { - const child = native.spawn(command, args, options); - child.once('close', () => { - closed = true; - }); - return child; - }); - let settled = false; - const call = createNodeProcessCapability({ workspaceDir: workspace, nativeProcesses: gated }) - .spawn(join(workspace, 'absent-executable'), [], {}, {}) - .catch((cause: unknown) => cause) - .finally(() => { - settled = true; + // The child's own `close` event cannot pin the owner join: the capability's handler only runs after it, so + // a publication that skipped `reservation.whenClosed` would still see a closed child (and, on POSIX with + // no helper, an already-retired owner). Each publication branch of `runChild` is therefore tested by + // gating the reservation's join itself and observing when the result reaches the caller. + const longLived = ['-e', 'setInterval(() => {}, 30000)']; + const branches = [ + { + name: 'failed-spawn', + command: () => join(workspace, 'absent-executable'), + args: [] as string[], + expected: { message: 'the command failed to run' }, + }, + { + name: 'natural-exit', + command: () => process.execPath, + args: ['-e', ''], + expected: { exitCode: 0 }, + }, + { + name: 'abort', + command: () => process.execPath, + args: longLived, + abort: true, + expected: { message: 'the command was aborted' }, + }, + { + name: 'timeout', + command: () => process.execPath, + args: longLived, + timeoutMs: 300, + expected: { message: 'the command timed out after 300ms' }, + }, + ] as const; + + it.each(branches)( + 'withholds the $name result until the owner reservation joins close', + async (branch) => { + const real = nativeProcessesForInvocation(createCliInvocation()); + if (real === undefined) throw new Error('explicit invocation missing'); + const gate = deferred(); + const gated: NativeProcessController = { + ...real, + reserve: async (kind) => ({ ...(await real.reserve(kind)), whenClosed: gate.promise }), + }; + const spawned = deferred(); + let closed = false; + vi.mocked(spawn).mockImplementationOnce((command, args, options) => { + const child = native.spawn(command, args, options); + child.once('close', () => { + closed = true; + }); + spawned.resolve(); + return child; }); - await vi.waitFor(() => expect(closed).toBe(true)); - // Microtasks queued by the close dispatch have run; give any ungated publication one more turn. - await new Promise((resolve) => setImmediate(resolve)); - expect(settled, 'PROCESS_OWNER_CLOSE_JOIN').toBe(false); - gate.resolve(); - const error = await call; - expect(error).toBeInstanceOf(ProcessCapabilityError); - expect(error).toMatchObject({ message: 'the command failed to run' }); - }); + const controller = new AbortController(); + let settled = false; + const call = createNodeProcessCapability({ workspaceDir: workspace, nativeProcesses: gated }) + .spawn( + branch.command(), + [...branch.args], + {}, + 'timeoutMs' in branch ? { timeoutMs: branch.timeoutMs } : {}, + controller.signal, + ) + .then( + (value: unknown) => value, + (cause: unknown) => cause, + ) + .finally(() => { + settled = true; + }); + if ('abort' in branch) { + await spawned.promise; + controller.abort(); + } + await vi.waitFor(() => expect(closed).toBe(true), { timeout: 3_000 }); + // Microtasks queued by the close dispatch have run; give any ungated publication one more turn. + await new Promise((resolve) => setImmediate(resolve)); + expect(settled, `PROCESS_OWNER_CLOSE_JOIN_${branch.name}`).toBe(false); + gate.resolve(); + expect(await call).toMatchObject(branch.expected); + }, + ); }); From ebbec1c05fab49b2b2e2f594c8eaee481194d4a3 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 19 Oct 2026 16:35:27 +0300 Subject: [PATCH 04/97] test(cli): harden the owner-join table and drop an ineffective wait bound Independent review of the follow-up found three weaknesses in the join table and one change that did not do what it claimed. - process-close.test.ts: the negative assertion waited one macrotask, so an ungated publication that was merely late (20 ms, 60 ms) slipped past; it now holds the gate for 100 ms and catches both in all four cases. A case that failed before the gate opened leaked an immortal node child (the abort mutant reproduced it); the case now reaps the child and releases the gate in a finally - native-forwarding.test-support.ts: restore the original default waits. The 3 s bound on the real-child boot wait could not help its only caller, whose outer waits in drive-home.test.ts use the 1 s default and enclose the boot, so a slow boot failed exactly as before, only later. No flake was ever reproduced (0 failures under heavy load), and the standard is wary of lengthening waits to hide one - mcp-orphan.e2e.test.ts: the skip comment said every case fails at the first execFileSync, but only one case reads the process table. It now says the whole suite is skipped on Windows out of caution because it has only run on POSIX Co-Authored-By: Claude Sonnet 5.5 --- .../native-forwarding.test-support.ts | 11 +------- .../engine/tool-host/process-close.test.ts | 28 +++++++++++++------ apps/cli/src/harness/mcp-orphan.e2e.test.ts | 6 ++-- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/apps/cli/src/commands/native-forwarding.test-support.ts b/apps/cli/src/commands/native-forwarding.test-support.ts index 3791569b..e58f100b 100644 --- a/apps/cli/src/commands/native-forwarding.test-support.ts +++ b/apps/cli/src/commands/native-forwarding.test-support.ts @@ -10,15 +10,6 @@ import { type NativeProcessController, } from '../process/native-process-owner.js'; -/** - * Applied to the ONE wait that observes a real child booting (its marker file): vitest's default - * `waitFor` timeout is 1 s, which a cold or loaded worker can exceed before a node child has started. - * It leaves 2 s of the 5 s default case timeout for setup; a failure after more than that is reported - * by the case timeout instead of the named assertion. The other waits here only observe registry - * reactions of an already-started or already-closed child and keep the default. - */ -const REAL_CHILD_BOOT_WAIT_MS = 3_000; - /** A physical child witness for host-only forwarding, with an independent tool timeout. */ export async function witnessNativeToolProcess( nativeProcesses: NativeProcessController | undefined, @@ -103,7 +94,7 @@ export async function holdNativeToolProcess( expect(existsSync(marker)).toBe(true); // physical native entry, not only a reserved field expect(invocation.registry.activeOwners).toBe(before + 1); }, - { interval: 1, timeout: REAL_CHILD_BOOT_WAIT_MS }, + { interval: 1 }, ); return { close }; } catch (error) { diff --git a/apps/cli/src/engine/tool-host/process-close.test.ts b/apps/cli/src/engine/tool-host/process-close.test.ts index e126c815..c43ddb2c 100644 --- a/apps/cli/src/engine/tool-host/process-close.test.ts +++ b/apps/cli/src/engine/tool-host/process-close.test.ts @@ -173,8 +173,10 @@ describe('native child resource lifetime after an error (ADR-0103)', () => { }; const spawned = deferred(); let closed = false; + let started: ChildProcess | undefined; vi.mocked(spawn).mockImplementationOnce((command, args, options) => { const child = native.spawn(command, args, options); + started = child; child.once('close', () => { closed = true; }); @@ -198,16 +200,26 @@ describe('native child resource lifetime after an error (ADR-0103)', () => { .finally(() => { settled = true; }); - if ('abort' in branch) { - await spawned.promise; + try { + if ('abort' in branch) { + await spawned.promise; + controller.abort(); + } + await vi.waitFor(() => expect(closed).toBe(true), { timeout: 3_000 }); + // Hold the gate for a fixed margin, not one macrotask: a publication that is merely late (a timer + // or a retried step) must be caught too, and a single turn let a 20 ms-late one through. + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(settled, `PROCESS_OWNER_CLOSE_JOIN_${branch.name}`).toBe(false); + gate.resolve(); + expect(await call).toMatchObject(branch.expected); + } finally { + // A failing case must not leave the long-lived child (or a pending call) behind. + gate.resolve(); controller.abort(); + if (started !== undefined && started.exitCode === null && started.signalCode === null) + started.kill('SIGKILL'); + await call; } - await vi.waitFor(() => expect(closed).toBe(true), { timeout: 3_000 }); - // Microtasks queued by the close dispatch have run; give any ungated publication one more turn. - await new Promise((resolve) => setImmediate(resolve)); - expect(settled, `PROCESS_OWNER_CLOSE_JOIN_${branch.name}`).toBe(false); - gate.resolve(); - expect(await call).toMatchObject(branch.expected); }, ); }); diff --git a/apps/cli/src/harness/mcp-orphan.e2e.test.ts b/apps/cli/src/harness/mcp-orphan.e2e.test.ts index 2e8d23b0..daf9076c 100644 --- a/apps/cli/src/harness/mcp-orphan.e2e.test.ts +++ b/apps/cli/src/harness/mcp-orphan.e2e.test.ts @@ -105,8 +105,10 @@ async function until(predicate: () => boolean, budgetMs = 10_000): Promise { From fcfe79b125ab233d4c786e4ec0930a2f549c9035 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 19 Oct 2026 16:35:27 +0300 Subject: [PATCH 05/97] docs(llm): correct overclaiming comments in the url scrub rule Comment-only: no behaviour, anchor or pinned count changes. - the URL rule docblock and the characterisation-test comment said the consequences "never reveal" a credential; a credential split by a plain space that also holds a raw @ stays the documented opaque-fragments-across- spaces limit, as it was for the pre-PR matcher - the wordBlocksStart comment now states that 'a' stands for every ASCII word character, which holds because each current boundary class rejects all of them or none - secrets-controls README: the R1 recovery mutant now only affects the key rules, since the URL and header rules supply their own joined patterns Co-Authored-By: Claude Sonnet 5.5 --- packages/llm/src/scrub-secrets.test.ts | 2 +- packages/llm/src/scrub-secrets.ts | 9 ++++++--- tools/secrets-controls/README.md | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/llm/src/scrub-secrets.test.ts b/packages/llm/src/scrub-secrets.test.ts index fb9c9121..c26de808 100644 --- a/packages/llm/src/scrub-secrets.test.ts +++ b/packages/llm/src/scrub-secrets.test.ts @@ -171,7 +171,7 @@ describe('finite diagnostic credential backstop regressions (PR #95 M2)', () => }); // Documents, and does not endorse, two consequences of a URL rule with no word boundary. Both fail - // closed: they blank or widen a diagnostic, they never reveal one. The whitespace-stripped residual probe + // closed: they blank or widen a diagnostic rather than reveal one. The whitespace-stripped residual probe // glues the prose before a URL to its scheme, and a longer scheme name matches by its suffix. it.each([ { diff --git a/packages/llm/src/scrub-secrets.ts b/packages/llm/src/scrub-secrets.ts index b547d979..2e587c78 100644 --- a/packages/llm/src/scrub-secrets.ts +++ b/packages/llm/src/scrub-secrets.ts @@ -32,9 +32,11 @@ interface SecretRule { * here would let any scheme glued to a preceding word character through unmasked. This PR added * ws/wss/ftp/ftps under the same rule. The rule has no `\b` prefix, so it supplies its own joined form * (the split-credential scanner would otherwise strip two characters that are not there) and its - * `boundary` blocks no start. Consequences, all fail-closed and pinned by tests: a longer scheme name + * `boundary` blocks no start. Consequences, pinned by tests and fail-closed: a longer scheme name * ending in one of these (`sftp://u@h`) matches by suffix, and the whitespace-stripped residual probe - * can turn prose followed by a URL and a later `@` into a whole-diagnostic `[REDACTED]`. + * can turn prose followed by a URL and a later `@` into a whole-diagnostic `[REDACTED]`. A credential + * split by a plain SPACE that also holds a raw `@` remains the documented opaque-fragments-across-spaces + * limit, as it was for the pre-PR matcher. */ const URL_USERINFO_PATTERN = /(?:https?|wss?|ftps?):\/\/[^/\s]*@/gi; @@ -135,7 +137,8 @@ function hasSplitCredential(text: string, joined: string, rule: SecretRule): boo // Native scanning avoids a JS/coverage branch for every byte of unrelated large text. if (!new RegExp(source, canonical.flags).test(joined)) return false; const pattern = new RegExp(source, canonical.flags.replace('g', 'y')); - // Only a rule whose boundary rejects ASCII word characters may skip word-preceded starts. + // Only a rule whose boundary rejects ASCII word characters may skip word-preceded starts. `'a'` stands + // for all of them: every current boundary class rejects all ASCII word characters or none of them. const wordBlocksStart = rule.boundary.test('a'); let originalOffset = 0; let joinedOffset = 0; diff --git a/tools/secrets-controls/README.md b/tools/secrets-controls/README.md index 20f9e206..e385c4f5 100644 --- a/tools/secrets-controls/README.md +++ b/tools/secrets-controls/README.md @@ -196,7 +196,7 @@ Two R1 original-boundary recovery controls add eight named assertions, retaining 51 prior identities and required witnesses. The core removal admits embedded-word starts while retaining the mapper's late refusal and bounded probe; the rejected candidate consumes the later valid start. The LLM removal replaces original-boundary checks with joined-boundary checks only -for the non-header rules in its unified scanner, preserving the finite-header path. All four +for the key (`sk-`/`AIza`) rules in its unified scanner (the URL and header rules supply their own joined patterns), preserving the finite-header path. All four OpenAI/Google × LF/TAB poisoned-prefix cases must then fail with `SECRETS_B2_ORIGINAL_BOUNDARY`; public embedded-word preservation and the later complete split credential are From 7913b7b7bb3f0d5db47c5327963c954b296c85d6 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 19 Oct 2026 17:24:22 +0300 Subject: [PATCH 06/97] docs(llm): correct three comments after the third review round Comment-only: no behaviour, anchor or pinned count changes. - the header docblock no longer attributes the whole-diagnostic redaction of prose next to a URL to the missing word boundary: the pre-PR matcher did the same for http(s), and dropping the boundary only extends it to prose that whitespace stripping glues to the scheme - the joinedPattern contract says the SyntaxError occurs on LF/TAB input that no earlier rule has already redacted, not on every such input - secrets-controls README: the R2 recovery mutant also selects the URL rule now that it has its own joined pattern, with no effect on its result Co-Authored-By: Claude Sonnet 5.5 --- packages/llm/src/scrub-secrets.ts | 8 +++++--- tools/secrets-controls/README.md | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/llm/src/scrub-secrets.ts b/packages/llm/src/scrub-secrets.ts index 2e587c78..6c057665 100644 --- a/packages/llm/src/scrub-secrets.ts +++ b/packages/llm/src/scrub-secrets.ts @@ -9,7 +9,7 @@ import { stripTerminalControls } from '@relavium/shared/terminal-safe'; * independently, including terminal-normalized and LF/TAB-joined views, before any * other rule can erase its identifying context. A match revealed by normalization masks the whole * diagnostic. The finite patterns check boundaries in the original text (the URL rule has none, like the - * pre-PR matcher, and so fails closed on prose that sits next to a URL) and preserve already-masked spans; + * pre-PR matcher, so a scheme glued to a word character is masked in place) and preserve already-masked spans; * this is not universal fragment discovery. Query names are excluded from the residual probe. * Pure and total for string input: regex replacements and a * finite residual check, with no parsing or external calls on the error path. @@ -34,7 +34,8 @@ interface SecretRule { * (the split-credential scanner would otherwise strip two characters that are not there) and its * `boundary` blocks no start. Consequences, pinned by tests and fail-closed: a longer scheme name * ending in one of these (`sftp://u@h`) matches by suffix, and the whitespace-stripped residual probe - * can turn prose followed by a URL and a later `@` into a whole-diagnostic `[REDACTED]`. A credential + * turns a path-less URL, prose and a later `@` into a whole-diagnostic `[REDACTED]` (the pre-PR matcher did + * so for http(s); dropping `\b` extends it to prose that stripping glues to the scheme). A credential * split by a plain SPACE that also holds a raw `@` remains the documented opaque-fragments-across-spaces * limit, as it was for the pre-PR matcher. */ @@ -120,7 +121,8 @@ function applyRule(text: string, rule: SecretRule): string { * All boundary-bearing rules use this scanner and their canonical matcher (or finite header-label * override). A rule that has a `joinedPattern` uses it as written; any other rule MUST begin its pattern * with `\b`, which is stripped here (deleting an explicit `joinedPattern` from a rule without `\b` - * throws on every LF/TAB input, which the suite catches at once). Prefixes start with ASCII letters; + * throws on any LF/TAB input that no earlier rule has already redacted, which the suite catches at once). + * Prefixes start with ASCII letters; * that prefilter skips only impossible starts. * Keys have one token run after a bounded prefix/minimum. Header labels are bounded and their * trailing whitespace cannot contain another label. URL authority excludes slash, so a later URL diff --git a/tools/secrets-controls/README.md b/tools/secrets-controls/README.md index e385c4f5..aa975388 100644 --- a/tools/secrets-controls/README.md +++ b/tools/secrets-controls/README.md @@ -213,7 +213,9 @@ all 53 prior control identities, paths, targets, markers and witness arrays. Fou `SECRETS_B2_HEADER_ORIGINAL_BOUNDARY`. The core removal admits an embedded outer label but retains the mapper's late boundary refusal, reproducing consumption of the valid inner label. The LLM removal substitutes joined-boundary checks only for -its finite-header rule; other rules still use original boundaries. Neither removal +its finite-header rule; other rules still use original boundaries. (The URL rule is selected +too, because it supplies its own joined pattern, but its boundary blocks no start, so the +substitution cannot change its result.) Neither removal disables bounded JWT probing or replaces canonical matcher authority. The earlier split-header-label control now removes the header helper's actual mask delivery. Core and LLM stress cases check bounded preparation with length/Boolean assertions, From 9902e0a0fec88755d90e34484bf4f024b21882c4 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 19 Oct 2026 18:05:58 +0300 Subject: [PATCH 07/97] docs(docs): record the post-merge review of a3e164b5 a3e164b5 (corrections from the third independent review of PR #95, including a secret-scrubbing rule) merged with author verification only. Record the independent review that followed and the open findings it left. - phase register: the merge checkpoint no longer claims the commit has no independent acceptance, and a new section records four review rounds and a security sitting (per-area results, the findings and their dispositions, what the reviewers did and did not cover, the follow-up commits) - deferred-tasks: the residuals the review found and did not fix, two of them security items that predate the follow-up (an api-key:Bearer token leak, and userinfo with a raw ? or # before the first colon left unmasked), plus the examined-and-unchanged waitFor bound and the review's nits - current.md, CLAUDE.md, AGENTS.md: the merge limits are stated as resolved or pointed at the open findings instead of "author verification only" Refs: ADR-0111 Co-Authored-By: Claude Sonnet 5.5 --- AGENTS.md | 14 +++-- CLAUDE.md | 14 +++-- docs/roadmap/current.md | 18 +++--- docs/roadmap/deferred-tasks.md | 10 ++- .../phase-2.5.5-hardening-and-remediation.md | 62 ++++++++++++++++--- 5 files changed, 89 insertions(+), 29 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6b36255f..55f6c271 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -118,12 +118,14 @@ joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 itself is NOT complete: Steps 9–19 have NOT STARTED and whole-wave acceptance remains Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) -records what merged, its review attributions, and one open limit: the final head `a3e164b5` -(corrections from a further independent review, including a secret-scrubbing rule) has -author verification only. CI for it and the merge commit finished green, and SonarCloud's -`main` gate (red since at least PR #90) was cleared on 2026-10-19. Next is Step 9 (release -map/custody substrate, #46/ADR-0111, split into 9a–9d in the increment table); new commits -on `development` start a new publication checkpoint. No provider, +records what merged and its review attributions. The final head `a3e164b5` (corrections from a +further independent review, including a secret-scrubbing rule) merged with author verification +only and was then independently reviewed (four rounds and a security sitting, with follow-up +fixes in later commits); CI for it and the merge commit finished green, and SonarCloud's `main` +gate (red since at least PR #90) was cleared on 2026-10-19. Open review findings, including two +security residuals, are in deferred-tasks.md. Next is Step 9 (release map/custody substrate, +#46/ADR-0111, split into 9a–9d in the increment table); new commits on `development` start a +new publication checkpoint. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. See [docs/roadmap/current.md](docs/roadmap/current.md) for live status. diff --git a/CLAUDE.md b/CLAUDE.md index 30f8ce78..045ea8b4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -207,12 +207,14 @@ joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 itself is NOT complete: Steps 9–19 have NOT STARTED and whole-wave acceptance remains Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) -records what merged, its review attributions, and one open limit: the final head `a3e164b5` -(corrections from a further independent review, including a secret-scrubbing rule) has -author verification only. CI for it and the merge commit finished green, and SonarCloud's -`main` gate (red since at least PR #90) was cleared on 2026-10-19. Next is Step 9 (release -map/custody substrate, #46/ADR-0111, split into 9a–9d in the increment table); new commits -on `development` start a new publication checkpoint. No provider, +records what merged and its review attributions. The final head `a3e164b5` (corrections from a +further independent review, including a secret-scrubbing rule) merged with author verification +only and was then independently reviewed (four rounds and a security sitting, with follow-up +fixes in later commits); CI for it and the merge commit finished green, and SonarCloud's `main` +gate (red since at least PR #90) was cleared on 2026-10-19. Open review findings, including two +security residuals, are in deferred-tasks.md. Next is Step 9 (release map/custody substrate, +#46/ADR-0111, split into 9a–9d in the increment table); new commits on `development` start a +new publication checkpoint. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. For live status, per-PR history, milestone dates, and open obligations, see the canonical diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index f8bedd2b..9ea8e237 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -147,15 +147,17 @@ diagnostics (2); secret provenance at public sinks, ADR-0113 (3); the typed erro engine tails (5); the pure process/discovery substrate, ADR-0110 (6); the lazy runtime, finite compiled graph and exact-tarball consumers, ADR-0111 (7); and creation-owned native process integration (8). -Three points are recorded rather than papered over; two are now resolved. **Open:** the merged head `a3e164b5` carries -corrections from a further independent review (one medium and three low findings: a secret-scrubbing production rule and -two behaviour-neutral import moves, plus tests and causal-control pins) that have **author verification only** — no -independent review rounds and no security sitting cover them. **Resolved:** CI had not finished when the merge happened -(the required `lint · typecheck · test` job was still running); the three runs later completed `success`, 15 of 15 jobs -each, as the [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#publication-state-observed-at-merge) +Three points were recorded rather than papered over; all three are now resolved. The merged head `a3e164b5` carried +corrections from a further independent review (a secret-scrubbing production rule, two behaviour-neutral import moves, +tests and causal-control pins) with **author verification only** at merge; a post-merge independent review — four +rounds and a security sitting, with fixes in follow-up commits — accepted them, as the +[review record](phases/phase-2.5.5-hardening-and-remediation.md#independent-review-of-a3e164b5-and-its-follow-up-corrections--2026-10-19) shows. CI had not finished when the merge +happened (the required `lint · typecheck · test` job was still running); the three runs later completed `success`, +15 of 15 jobs each, as the [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#publication-state-observed-at-merge) records. And SonarCloud's quality gate, which had failed on `main` since at least PR #90 because its dispositions lived -only on PR branches, was cleared on 2026-10-19 (zero open vulnerabilities, gate OK). The review's remaining low and nit -findings are unactioned, in [deferred-tasks.md](deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18). +only on PR branches, was cleared on 2026-10-19 (zero open vulnerabilities, gate OK). Open findings remain, including two +security residuals that predate the follow-up — an `api-key:Bearer ` token leak and unmasked userinfo with a raw +`?`/`#` — in [deferred-tasks.md](deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18). ## Next work — Wave 3 — Clear the ground diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index f97f8739..44a8bcd8 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -390,6 +390,14 @@ from that review and were not re-verified by the author; items marked *(author)* `ci.yml` and the Sonar backlog section below), the S4036 accepted as on the PR #95 branch. SonarCloud then reported zero open vulnerabilities and an OK gate for `main`. No rule, threshold or gate was changed; the per-issue record is local and Git-ignored. *(was medium · process)* +- [ ] **An `api-key` header label erases the `Bearer`/`Basic` label that follows it, so the token leaks.** *Low (security).* *(author)* `scrubSecrets('api-key:Bearer abcdefghijkl123456')` returns `api-key:[REDACTED] abcdefghijkl123456` — the `api-key` rule consumes the word `Bearer` as the header's value and the Bearer rule then has no label to anchor on. The pre-PR matcher returned `api-key:Bearer [REDACTED]`. Surfaced by the post-merge security sitting, reproduced independently; it comes from the PR's rule order, not from `a3e164b5`, and is **not fixed**. A plain `Authorization: Bearer …` is unaffected. Fix idea: let the header value skip a following `Bearer`/`Basic` label, or run those rules first. *(low · packages/llm/src/scrub-secrets.ts)* +- [ ] **URL userinfo containing a raw `?` or `#` before the first `:` (or with no `:`) is not masked.** *Low (security).* *(author, from two independent differential runs)* `https://us?er:TOKEN@h/`, `http://#TOKEN@h` and `https://my.user:1234?TOKEN@h/` pass through unchanged; the pre-PR matcher masked all of them. It follows from the deliberate host-query policy in `maskUrlUserinfo` (a raw `?`/`#` may begin public query text, e.g. `https://host?contact=a@b.c`) and the colon-first cases are pinned by tests; it is identical at `19c99968` and is not introduced by the follow-up. **A maintainer call** whether a username shaped that way is worth the false positives. Fix idea: exempt only when the text before the `?`/`#` is non-empty and host-shaped (contains `.`, a `:port`, `localhost` or a bracketed IPv6), otherwise mask. *(low · packages/llm/src/scrub-secrets.ts)* +- [ ] **Credentials in URLs of other schemes are outside the finite rule.** *Low, informational.* `postgres://`, `redis://`, `mongodb(+srv)://`, `ssh://` userinfo, malformed `https:/u:p@h`, a raw `/` inside a password, and invisible characters inside the scheme are not masked, identically before and after the follow-up. The file states its shapes are finite; a database connection error is the realistic carrier, so the scheme list may deserve a decision. *(info · packages/llm/src/scrub-secrets.ts)* +- [ ] **The URL rule's split-credential scan costs 2.1–2.5× on letter-heavy input, and nothing times that loop.** *Low / nit.* With `NO_BOUNDARY` the scanner tests every ASCII letter start for the URL rule; at 8 MiB the worst case stays under about 1.1 s and linear (1–16 MiB scaling checked). A prefilter on `h`/`w`/`f` would recover most of it. The existing 8 MiB timing case contains no `@`, so it returns before the loop; a guard would use a leading `a\nhttps://u@h ` plus 8 MiB of letters, but see the load-sensitive timing item below before adding another wall-clock assertion. SonarCloud was not run on the never-matching `NO_BOUNDARY = /(?!)/`; check its next analysis. *(low · packages/llm/src/scrub-secrets.ts)* +- [ ] **Two latent invariants in `hasSplitCredential` are unchecked.** *Nit.* A boundary-bearing rule with neither `\b` nor a `joinedPattern` makes `scrubSecrets` throw a `SyntaxError` on LF/TAB input no earlier rule has redacted (the suite fails at once, so it is caught, but it would break `makeLlmError`'s no-throw claim); and `wordBlocksStart` tests `boundary.test('a')` as a proxy for "rejects every ASCII word character", true for all current boundaries. An assertion at module load would pin both. *(nit · packages/llm/src/scrub-secrets.ts)* +- [ ] **The userinfo control's joined-view half is not separately witnessed.** *Low (coverage).* *(author, from an independent run)* Mutating only `joinedPattern` of the URL rule leaves 111/111 tests green. The whitespace-stripped residual probe independently catches the class the existing cases use, so it may be largely an equivalent mutant; an added case that did not discriminate it was reverted rather than kept. No causal control pins the glued-scheme behaviour either — only ordinary tests do. *(low · tools/secrets-controls/mutations.mjs)* +- [ ] **`LOW-2` (the 1 s default `waitFor` on a real child) was examined and deliberately not changed.** *Low.* The only wait that observes a real child booting is `holdNativeToolProcess`'s marker wait, whose sole caller sits inside `drive-home.test.ts` waits (lines 633 and 637) that use the 1 s default and enclose the boot, so a longer inner bound cannot help; a retune must raise those outer waits together. No failure was reproduced (0 in dozens of loaded runs). `testing.md` also states that no readiness or case deadline is extended to pass the workspace gate. **Trigger: a recorded boot-time failure.** *(low · apps/cli/src/commands/native-forwarding.test-support.ts, apps/cli/src/home/drive-home.test.ts)* +- [ ] **Join-table test nits.** *Nit.* The `process-close.test.ts` gate margin is nominally 100 ms but `waitFor`'s 50 ms poll makes it 100–150 ms, so a publication more than about 150 ms late is invisible; the three rejection cases assert the message but not the `ProcessCapabilityError` class; the `mcp-orphan.e2e.test.ts` comment's statement that its cases have only ever run on POSIX is historical and unverifiable; the abort case's `await spawned.promise` is unbounded (a spawn that never happens fails only as the 5 s case timeout), its closed-child `waitFor` carries no tag, and the 100/300/3,000 ms margins are bare literals. *(nit · apps/cli/src/engine/tool-host/process-close.test.ts, apps/cli/src/harness/mcp-orphan.e2e.test.ts)* - [ ] **Native-lifetime watchdogs are fixed 5 s bounds inside 10 s cases (`LOW-1`).** *Low.* The watchdogs in `apps/cli/src/process/native-process-owner.test.ts`, `engine/tool-host/os-native-lifetime.test.ts`, `render/editor-native-lifetime.test.ts`, `render/fixtures/native-editor.mjs` and @@ -402,7 +410,7 @@ from that review and were not re-verified by the author; items marked *(author)* - [ ] **`bounds admitted-size failed header and URL preparation` is load-sensitive.** *Low.* *(author)* It allows 5 s for two 8 MiB scrubs that take about 0.75 s in total, and it failed twice with the load average near 65 (6.4 s in the run that logged the duration). Measured alternately, the pre-change code and the corrected code take the same - time, so the failures were load, not the correction. The test + time, so the failures were load, not the correction. Two independent reviewers saw it too (5,590 ms and 3,453 ms at load averages near 88–97) and recorded that `tools/secrets-controls/README.md` says the fixture runs without a machine-speed threshold, which this assertion contradicts; six controls run this file, so a loaded runner can turn a fixed phase red. The test guards against a quadratic regression, so any loosening must stay far below minutes. **Trigger: the next CI failure of this case.** *(low · packages/llm/src/scrub-secrets.test.ts, `SECRETS_B2_HEADER_PREPARATION`)* - [ ] **Raw exit-code literals where `EXIT_CODES` names exist.** *Nit.* *(reviewer)* reported diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index ac787db1..0d53c9e1 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -9184,7 +9184,8 @@ Detailed review records remain local and Git-ignored. The difference between `ec0f3e10` and the merged head is exactly twelve non-documentation paths plus three documentation files. Two test files are the independently accepted fixture -increments above. The other ten are the commit below, which has **no independent acceptance**. +increments above. The other ten are the commit below. It merged with author verification only; its +post-merge independent review is recorded in [its own section](#independent-review-of-a3e164b5-and-its-follow-up-corrections--2026-10-19) after this checkpoint. #### Independent review of the published head and its corrections @@ -9199,25 +9200,26 @@ before acting. Corrections landed as `a3e164b5`: | Finding | Disposition | | --- | --- | | **MEDIUM-1** — URL userinfo not masked when its scheme is glued to a word character (`xhttps://user:pass@host`) | Reproduced at `19c99968` through both `scrubSecrets` and the CLI sanitizer. Fixed in `packages/llm/src/scrub-secrets.ts`: the rule returns to its pre-PR word-adjacency (no `\b`) and supplies its own joined pattern, because the split-credential scanner otherwise strips two characters from a pattern that has none. Fourteen new `scrub-secrets` cases and one CLI case; reverting the production change fails all fourteen (the CLI case failed against the stale build before it was rebuilt). | -| **LOW-4** — failed-spawn test does not pin the native close join | Confirmed by mutation: removing `reservation.whenClosed` left all three `process-close` tests green. The reviewer's suggested `activeOwners === 0` assertion was also tried and **did not** kill that mutant, because the owner retires synchronously inside its own `close` handler on POSIX. Replaced by a case that gates the reservation's `whenClosed` and observes publication; it fails only when the join is removed. | -| **LOW-2** — default 1 s `vi.waitFor` on a real spawned child | Fixed: the four waits in `native-forwarding.test-support.ts` carry an explicit 4 s bound, below the 5 s default case timeout. | +| **LOW-4** — failed-spawn test does not pin the native close join | Confirmed by mutation: removing `reservation.whenClosed` left all three `process-close` tests green. The reviewer's suggested `activeOwners === 0` assertion was also tried and **did not** kill that mutant, because the owner retires synchronously inside its own `close` handler on POSIX. Replaced by a case that gates the reservation's `whenClosed` and observes publication; it fails only when the join is removed. The post-merge review extended it to all four publication branches. | +| **LOW-2** — default 1 s `vi.waitFor` on a real spawned child | Added in `a3e164b5` (an explicit 4 s bound on four waits), then **reverted** after the post-merge review: the only wait that observes a real child booting is enclosed by 1 s waits in its sole caller, so a longer inner bound could not help. Examined and deliberately unchanged; see deferred-tasks.md. | | **LOW-3** — POSIX `ps` suite not skipped on Windows | Fixed: `describe.skipIf(process.platform === 'win32')`. Prettier re-indented the suite body, so the raw diff of that file is 215 lines; `git show -w` shows the 7-insertion / 2-deletion semantic change. | | **LOW-1** — fixed 5 s native watchdogs | **Not changed.** Three runs of the three affected native files (52 tests) all passed at a load average of 80–96, the reviewer could not observe the Windows root cause, and the constants are coupled to neighbouring bounds, so a retune would be speculative. Recorded in [deferred-tasks.md](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18). | | Nit — import above the module docblock in `parser.ts`, `agent-parser.ts` | Fixed (no behaviour change). | | Nit — `packages/shared` curated export list | Closed without change: it is an illustrative existence sample, not an exhaustive contract. | | Other nits and info items | Open, in the same deferred-tasks section. | -The production change restores a pre-PR behavior, but it touches a secret-scrubbing rule and +The production change restores the pre-PR word-adjacency for http(s) (and gives the ws/wss/ftp/ftps +schemes this PR added the same behaviour), but it touches a secret-scrubbing rule and the repository's causal controls: the `diagnostic-raw-password-userinfo` source anchor moved to the new `URL_USERINFO_PATTERN` constant, and the pinned complete workloads changed (89→103, -179→180, 277→278) because the files gained cases. **Verification is the author's only.** The +179→180, 277→278) because the files gained cases. **At merge, verification was the author's only.** The full local gate was run in two parts: `pnpm run ci` through the workspace tests, builds, format, fences, bundle closure and smokes; then, after the anchor/pin update, `test:acceptance-tools` (633 of 633) and the secrets, wire, usage and filesystem mutation gates (exit code 0 each, the userinfo control 103 / 98 with five failing cases / 103), plus tools lint, typecheck and format and the affected suites under `CI=true GITHUB_ACTIONS=true`. A second complete `pnpm run ci` after -the pin update and a coverage run were not performed. No independent review rounds and no security -sitting cover `a3e164b5`; none is claimed. +the pin update and a coverage run were not performed. No independent review rounds or security +sitting had covered `a3e164b5` at merge; the review that followed is recorded after this checkpoint. #### Publication state observed at merge @@ -9263,7 +9265,7 @@ check-run attached to the merge commit itself is historical and unchanged. Recor **Wave 3 Steps 1–8 are on `main`; Wave 3 remains in progress.** Open after this merge: Steps 9–19 and whole-wave acceptance (Step 19: A/E/G and M2.5.5-3 evidence, promoting core's ≥90% line/branch floor to required CI, the reconciled EXIT:4 reading, two new whole-wave cumulative reviews); -an independent review of the `a3e164b5` delta; and the unactioned review findings. Phase 2.5.5 exit criteria 4–8, sub-stream I, later +and the unactioned review findings. Phase 2.5.5 exit criteria 4–8, sub-stream I, later product waves and the existing approved deferrals keep their canonical homes. The merge adds no deferral, ADR or release. @@ -9274,3 +9276,47 @@ archive/crypto security sitting. The maintainer-approved 9a–9d subdivision is [increment table](#proposed-implementation-increments), and 9a is first. Steps 10–19 follow the approved plan in order. New implementation commits on `development` now start a new publication checkpoint; PR #95 no longer bounds their review scope. + +### Independent review of `a3e164b5` and its follow-up corrections — 2026-10-19 + +The merged delta `19c99968..a3e164b5` had author verification only. After the merge it was reviewed by +independent reviewers, each in an isolated worktree at an exact tree, given the code and a neutral statement of +purpose, never the author's conclusions: three area reviewers (the scrubbing rule and its tests; the +causal-control pins and README; the CLI test changes and two import moves) and a separate security sitting on +the rule. Later rounds review the **cumulative** result, so the fixes made between rounds are reviewed too. Detailed +records stay local and Git-ignored. + +| Round | Source | Result | +| --- | --- | --- | +| 1 | `a3e164b5` (tree `415af966`) | All four NONCLEAN: no blocker, high or medium; low and nit findings. | +| 2 | `f597977a` (tree `7fd14b05`) | Rule and tests **CLEAN**, pins and README **CLEAN**, security sitting **CLEAN**; CLI tests NONCLEAN (one medium, three low). | +| 3 | `fcfe79b1` (tree `b45d510d`) | Rule and tests **CLEAN**, pins and README **CLEAN**, CLI tests **CLEAN**. | +| 4 | `fcfe79b1` | CLI tests **CLEAN**, a second fresh reviewer on the same bytes. | + +Each area therefore has two clean rounds on identical code bytes (rule and tests: rounds 2–3; pins and README: +2–3; CLI tests: 3–4), plus one clean security sitting (round 2). Differential corpora ran the pre-PR matcher, +`19c99968`, `a3e164b5` and the corrected tree side by side: the scrubbing reviewers used 0.8–1.1 million structured +inputs and up to 2.2 million random strings, the security sitting 48 million exhaustive and 31.5 million random +token sequences with a planted-token leak oracle (which did detect leaks in the three older versions). **Against +`19c99968` no input regressed, apart from accidents of that version:** the few inputs it redacted wholesale only +because its raw and normalized views disagreed (for example an ANSI-coloured URL whose credential is split by a +plain space) are now masked in place and keep the documented opaque-fragments-across-spaces limit that the +pre-PR matcher shares. **Against the pre-PR matcher** the only inputs it masked and the corrected tree does not +are the raw-`?`/`#` userinfo class recorded in deferred-tasks.md. Reviewers ran on macOS only, and no review ran SonarCloud +on the final code or exercised Windows. Three comment-only corrections in `scrub-secrets.ts` and two README +sentences were made after the last review of those files; they change no behaviour, anchor or count. + +| Finding (source) | Disposition | +| --- | --- | +| A glued scheme with an LF/TAB-split credential that also holds a raw `@` left a partial mask with part of the credential visible, e.g. `xhttps://user:p@ss\tword@host/v1` (security sitting, round 1; low). | **Fixed** in `c64f3c39`: the URL rule's `boundary` blocks no start, and the scanner's word-character shortcut applies only when a rule's boundary rejects word characters. The anchored scanner gate and pattern line are untouched, so no causal-control anchor moves. Two regression cases (reverting the gate fails exactly those) and CLI assertions. | +| The join of the owner reservation was pinned on the failed-spawn branch only; a mutant keeping it there alone passed every `process*` test (round 1; low). | **Fixed** in `f597977a` and `ebbec1c0`: one table over failed-spawn, natural exit, abort and timeout, mutation-verified branch by branch, against a stop-based join, against ungated publications up to about 100 ms late, and with the child reaped in a `finally` (removing it leaks a child). | +| The 4 s `waitFor` bound could not help its only caller (round 2; medium). | **Reverted** in `ebbec1c0`; `native-forwarding.test-support.ts` is byte-identical to its `19c99968` form. Examined and deliberately unchanged. | +| Stale pinned counts and README numbers (round 1; low). | **Fixed**: scrub-secrets set 89→111, sanitize + render-error 179→180, llm-error + sanitize + render-error 277→278; the README carries them and 25 files / 1,343 cases, measured independently by three reviewers. | +| Inaccurate or overclaiming comments (rounds 1–3). | **Corrected**: ws/wss/ftp/ftps were this PR's additions, not pre-PR behaviour; "never reveal" and "all fail-closed"; the `slice(2)` contract; the `'a'` proxy; the `mcp-orphan` skip reason. | +| Whole-diagnostic redaction of prose next to a path-less URL and a later `@`, and suffix matches such as `sftp://` (rounds 1–3; low). | **Characterised, not changed**: three tests and a docblock state the trade-off. It fails closed, equals the pre-PR matcher for http(s), and is new for the schemes this PR added. | +| Residuals that predate the follow-up and are not in this delta: an `api-key:Bearer ` token leak, userinfo with a raw `?`/`#` before the first `:`, URL schemes outside the finite rule, a constant-factor scan cost, two unchecked invariants, an equivalent-mutant coverage gap, the load-sensitive wall-clock test and join-table nits. | **Open, not fixed**: recorded in [deferred-tasks.md](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18). | + +Author verification of the corrected tree: full local `pnpm run ci` passes end to end at `7913b7b7` (workspace lint, typecheck, tests and build; format; fences; bundle closure; smokes; the acceptance tools; and the wire, usage, filesystem and secrets mutation families); the affected suites pass with and +without `CI=true GITHUB_ACTIONS=true`. The `a3e164b5` subject line (`refactor(tests)`, a non-package scope on a +change that spans llm, core, cli and tools) is already published and is not rewritten. The follow-up commits are +`c64f3c39`, `f597977a`, `ebbec1c0`, `fcfe79b1` and `7913b7b7`. From 38e10d42e41264bf40ec347a4de420fcc5907253 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 19 Oct 2026 22:01:13 +0300 Subject: [PATCH 08/97] docs(docs): record the wave 3 steps 9-19 preflight rulings A pre-implementation audit read the canonical documents for Wave 3 Steps 9-19 and the open PR #95 findings against the current source (sixteen auditors, fifteen adversarial verifiers and a completeness critic). The maintainer approved every recommendation on 2026-10-19. No new ADR is required. - Record the rulings, the PR95 plan amendment (scrubber residuals before 9a) and the recorded splits for Steps 9-19 in the phase register's increment table (29 increments, ID coverage unchanged apart from PR95). - Give every open PR #95 item an owner in deferred-tasks.md; LOW-1 and LOW-2 become approved deferrals with their triggers. - Reconcile stale status: the phase banner, the a3e164b5 review wording in current.md, where the post-merge follow-ups live (development, pushed, push CI 39275223233 green), README and CLAUDE/AGENTS status lines. - Add forward pointers from the preflight tables to the approval, a dated #45 note for ADR-0110's fixed infrastructure diagnostic, G30's pull forward to Step 13 in both phase files, and the Outcomes bullet's reconciliation with exit criterion 4. - Mark accepted Steps 1-8 task bullets DONE or certified. - AGENTS.md: describe the development-to-main PR branch model in use. Refs: ADR-0110, ADR-0111, ADR-0112, ADR-0113 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 14 +- CLAUDE.md | 11 +- README.md | 7 +- docs/roadmap/current.md | 21 ++- docs/roadmap/deferred-tasks.md | 41 +++--- .../phase-2.5.5-hardening-and-remediation.md | 137 +++++++++++++++--- .../phase-2.6-conversational-authoring.md | 5 + 7 files changed, 174 insertions(+), 62 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 55f6c271..161cf60b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -121,11 +121,12 @@ Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-re records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification only and was then independently reviewed (four rounds and a security sitting, with follow-up -fixes in later commits); CI for it and the merge commit finished green, and SonarCloud's `main` -gate (red since at least PR #90) was cleared on 2026-10-19. Open review findings, including two -security residuals, are in deferred-tasks.md. Next is Step 9 (release map/custody substrate, -#46/ADR-0111, split into 9a–9d in the increment table); new commits on `development` start a -new publication checkpoint. No provider, +fixes on `development`, not yet on `main`); CI for it and the merge commit finished green, and SonarCloud's `main` +gate (red since at least PR #90) was cleared on 2026-10-19. A pre-implementation review of Steps 9–19 +and the maintainer's [2026-10-19 rulings](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) +give every open PR #95 finding an owner and record 29 increments (no new ADR). Next is the PR95 +increment (the scrubber residuals, including both security residuals), then Step 9a.1 (release +map/custody substrate, #46/ADR-0111). `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. See [docs/roadmap/current.md](docs/roadmap/current.md) for live status. @@ -172,7 +173,8 @@ Never `npm` or `yarn`. `workspace:*` for inter-package deps; no circular deps. ## Git workflow -- Trunk-based on `main`; short-lived feature branches via PR. +- Work lands on the long-lived `development` branch and reaches the protected `main` (required CI + checks) through a PR from `development`. Do not create other branches unless the maintainer asks. - Conventional Commits with a per-package scope (`feat(core):`, `fix(llm):`, `docs(decisions):`). Reference the ADR or task the change advances. See [docs/standards/commit-style.md](docs/standards/commit-style.md). diff --git a/CLAUDE.md b/CLAUDE.md index 045ea8b4..b8350fb4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -210,11 +210,12 @@ Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-re records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification only and was then independently reviewed (four rounds and a security sitting, with follow-up -fixes in later commits); CI for it and the merge commit finished green, and SonarCloud's `main` -gate (red since at least PR #90) was cleared on 2026-10-19. Open review findings, including two -security residuals, are in deferred-tasks.md. Next is Step 9 (release map/custody substrate, -#46/ADR-0111, split into 9a–9d in the increment table); new commits on `development` start a -new publication checkpoint. No provider, +fixes on `development`, not yet on `main`); CI for it and the merge commit finished green, and SonarCloud's `main` +gate (red since at least PR #90) was cleared on 2026-10-19. A pre-implementation review of Steps 9–19 +and the maintainer's [2026-10-19 rulings](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) +give every open PR #95 finding an owner and record 29 increments (no new ADR). Next is the PR95 +increment (the scrubber residuals, including both security residuals), then Step 9a.1 (release +map/custody substrate, #46/ADR-0111). `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. For live status, per-PR history, milestone dates, and open obligations, see the canonical diff --git a/README.md b/README.md index c28b08d6..c1ae9b40 100644 --- a/README.md +++ b/README.md @@ -158,9 +158,10 @@ Relavium is under active development. The **CLI is published as v0.1.1**; the pu agent-session entry point, workflow runtime, inbound MCP client, and CLI management surface are implemented. **Phase 2.6.5 — Core Reliability Remediation is complete with approved deferrals**, with W8 merged in PR #92. The joint Phase 2.5.5/2.6 security remediation -Wave 2 merged to `main` on 2026-10-15 as PR #94. The next -engineering wave is “Clear the ground”: CLI safety, shared guards and decomposition -of the hot files needed by later product work. +Wave 2 merged to `main` on 2026-10-15 as PR #94. The current +engineering wave is “Clear the ground” (CLI safety, shared guards and decomposition +of the hot files needed by later product work); its first eight steps merged to `main` +on 2026-10-18 as PR #95. Status changes quickly, so this README intentionally stays high-level. The canonical source for the exact active wave, completed work, and open reliability obligations is diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index 9ea8e237..a64498b0 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -161,12 +161,15 @@ security residuals that predate the follow-up — an `api-key:Bearer ` to ## Next work — Wave 3 — Clear the ground -Wave 3 implementation continues on `development`: Steps 1–8 are merged, **Step 9 is next** — the release -map/custody substrate (#46, ADR-0111), split into 9a–9d in the [increment table](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments), -which carries a dedicated archive/crypto security sitting. Steps 10–19 -follow the approved [nineteen-increment plan](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) -and end with whole-wave acceptance at Step 19. New implementation commits start a new publication checkpoint; PR #95 -no longer bounds their review scope. See the [Wave 3 queue](#wave-3--clear-the-ground). +Wave 3 implementation continues on `development`: Steps 1–8 are merged. A pre-implementation document/source review +of Steps 9–19 and the maintainer's rulings are recorded in the +[2026-10-19 rulings](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) +(no new ADR; 29 recorded increments). **Next is the PR95 increment** — the PR #95 scrubber residuals, including the two security +residuals — and then **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), whose +9a.1–9d increments carry a dedicated archive/crypto security sitting. Steps 10–19 follow the +[increment table](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) and end with +whole-wave acceptance at Step 19. `development` is pushed after each accepted increment; PR #95 no longer bounds the +review scope of new commits. See the [Wave 3 queue](#wave-3--clear-the-ground). ## Historical W7 implementation checkpoints @@ -1144,8 +1147,10 @@ the [second corrective source-review completion](phases/phase-2.5.5-hardening-an (accepted at `ec0f3e10`), the [native Windows admission preparation](phases/phase-2.5.5-hardening-and-remediation.md#pr-95-native-windows-admission-preparation--2026-10-18) and the [supplemental fixture acceptance](phases/phase-2.5.5-hardening-and-remediation.md#pr-95-supplemental-native-fixture-source-acceptance--2026-10-18). Those scopes are not widened: the delta from `ec0f3e10` to the merged head is two independently accepted test-fixture -increments, documentation and the independent review's corrections at `a3e164b5`, which have author verification -only. Whole-wave acceptance remains Step 19. No new decision, credential, billing or provider capture gate is introduced. +increments, documentation and the independent review's corrections at `a3e164b5`, which merged with author +verification only and were then independently reviewed after the merge +([review record](phases/phase-2.5.5-hardening-and-remediation.md#independent-review-of-a3e164b5-and-its-follow-up-corrections--2026-10-19)); +the follow-up corrections are on `development`, not yet on `main`. Whole-wave acceptance remains Step 19. No new decision, credential, billing or provider capture gate is introduced. Decompose, guard and normalize every hot file a 2.6 workstream is about to land on. **The multi-file queue is Step 1 and runs first.** The approved diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index 44a8bcd8..f97a4290 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -2,7 +2,7 @@ > Status: Living -> Last updated: 2026-10-19 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Steps 9–19 not started). The independent PR #95 review's unactioned findings are recorded below. Existing approved deferrals remain. +> Last updated: 2026-10-19 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Steps 9–19 not started). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings. Existing approved deferrals remain. > The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) records completed prerequisites; full M/N lifecycle and other scheduled work remain with their owners. > > Earlier triage: the Phase-2.6 rewrite triaged every open item, the 2026-07-19 full-project @@ -373,9 +373,12 @@ Severity is the review's verified rating. Check an item off in the PR that resol An independent systematic review of [PR #95](https://github.com/HodeTech/Relavium/pull/95) at `19c99968` reported no blocker or high finding. Its medium and three of its four low findings were corrected in `a3e164b5` ([disposition table](phases/phase-2.5.5-hardening-and-remediation.md#independent-review-of-the-published-head-and-its-corrections)); -what follows was **not** actioned. These are `- [ ]` (unscheduled, actionable) items, not approved deferrals: -no maintainer ruling is recorded for them. Items marked *(reviewer)* are quoted -from that review and were not re-verified by the author; items marked *(author)* were measured or counted here. +what follows was **not** actioned in that pass. **Owner rulings, 2026-10-19:** the maintainer approved an owner for +every open item ([rulings](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19)). +Each `- [ ]` below is now **Scheduled** to a Wave 3 increment (the PR95 increment runs before Step 9a) and stays unchecked until +that increment lands, except LOW-1 and LOW-2, which are approved deferrals with their recorded triggers. Items +marked *(reviewer)* are quoted from that review and were not re-verified by the author; items marked *(author)* were +measured or counted here. - **DONE 2026-10-19 — SonarCloud's quality gate was red on `main` after every merge.** *(author)* The `main` analysis failed `new_security_rating` (C, required A) against the `previous_version` baseline (2026-06-04) because nine @@ -390,15 +393,15 @@ from that review and were not re-verified by the author; items marked *(author)* `ci.yml` and the Sonar backlog section below), the S4036 accepted as on the PR #95 branch. SonarCloud then reported zero open vulnerabilities and an OK gate for `main`. No rule, threshold or gate was changed; the per-issue record is local and Git-ignored. *(was medium · process)* -- [ ] **An `api-key` header label erases the `Bearer`/`Basic` label that follows it, so the token leaks.** *Low (security).* *(author)* `scrubSecrets('api-key:Bearer abcdefghijkl123456')` returns `api-key:[REDACTED] abcdefghijkl123456` — the `api-key` rule consumes the word `Bearer` as the header's value and the Bearer rule then has no label to anchor on. The pre-PR matcher returned `api-key:Bearer [REDACTED]`. Surfaced by the post-merge security sitting, reproduced independently; it comes from the PR's rule order, not from `a3e164b5`, and is **not fixed**. A plain `Authorization: Bearer …` is unaffected. Fix idea: let the header value skip a following `Bearer`/`Basic` label, or run those rules first. *(low · packages/llm/src/scrub-secrets.ts)* -- [ ] **URL userinfo containing a raw `?` or `#` before the first `:` (or with no `:`) is not masked.** *Low (security).* *(author, from two independent differential runs)* `https://us?er:TOKEN@h/`, `http://#TOKEN@h` and `https://my.user:1234?TOKEN@h/` pass through unchanged; the pre-PR matcher masked all of them. It follows from the deliberate host-query policy in `maskUrlUserinfo` (a raw `?`/`#` may begin public query text, e.g. `https://host?contact=a@b.c`) and the colon-first cases are pinned by tests; it is identical at `19c99968` and is not introduced by the follow-up. **A maintainer call** whether a username shaped that way is worth the false positives. Fix idea: exempt only when the text before the `?`/`#` is non-empty and host-shaped (contains `.`, a `:port`, `localhost` or a bracketed IPv6), otherwise mask. *(low · packages/llm/src/scrub-secrets.ts)* -- [ ] **Credentials in URLs of other schemes are outside the finite rule.** *Low, informational.* `postgres://`, `redis://`, `mongodb(+srv)://`, `ssh://` userinfo, malformed `https:/u:p@h`, a raw `/` inside a password, and invisible characters inside the scheme are not masked, identically before and after the follow-up. The file states its shapes are finite; a database connection error is the realistic carrier, so the scheme list may deserve a decision. *(info · packages/llm/src/scrub-secrets.ts)* -- [ ] **The URL rule's split-credential scan costs 2.1–2.5× on letter-heavy input, and nothing times that loop.** *Low / nit.* With `NO_BOUNDARY` the scanner tests every ASCII letter start for the URL rule; at 8 MiB the worst case stays under about 1.1 s and linear (1–16 MiB scaling checked). A prefilter on `h`/`w`/`f` would recover most of it. The existing 8 MiB timing case contains no `@`, so it returns before the loop; a guard would use a leading `a\nhttps://u@h ` plus 8 MiB of letters, but see the load-sensitive timing item below before adding another wall-clock assertion. SonarCloud was not run on the never-matching `NO_BOUNDARY = /(?!)/`; check its next analysis. *(low · packages/llm/src/scrub-secrets.ts)* -- [ ] **Two latent invariants in `hasSplitCredential` are unchecked.** *Nit.* A boundary-bearing rule with neither `\b` nor a `joinedPattern` makes `scrubSecrets` throw a `SyntaxError` on LF/TAB input no earlier rule has redacted (the suite fails at once, so it is caught, but it would break `makeLlmError`'s no-throw claim); and `wordBlocksStart` tests `boundary.test('a')` as a proxy for "rejects every ASCII word character", true for all current boundaries. An assertion at module load would pin both. *(nit · packages/llm/src/scrub-secrets.ts)* -- [ ] **The userinfo control's joined-view half is not separately witnessed.** *Low (coverage).* *(author, from an independent run)* Mutating only `joinedPattern` of the URL rule leaves 111/111 tests green. The whitespace-stripped residual probe independently catches the class the existing cases use, so it may be largely an equivalent mutant; an added case that did not discriminate it was reverted rather than kept. No causal control pins the glued-scheme behaviour either — only ordinary tests do. *(low · tools/secrets-controls/mutations.mjs)* -- [ ] **`LOW-2` (the 1 s default `waitFor` on a real child) was examined and deliberately not changed.** *Low.* The only wait that observes a real child booting is `holdNativeToolProcess`'s marker wait, whose sole caller sits inside `drive-home.test.ts` waits (lines 633 and 637) that use the 1 s default and enclose the boot, so a longer inner bound cannot help; a retune must raise those outer waits together. No failure was reproduced (0 in dozens of loaded runs). `testing.md` also states that no readiness or case deadline is extended to pass the workspace gate. **Trigger: a recorded boot-time failure.** *(low · apps/cli/src/commands/native-forwarding.test-support.ts, apps/cli/src/home/drive-home.test.ts)* -- [ ] **Join-table test nits.** *Nit.* The `process-close.test.ts` gate margin is nominally 100 ms but `waitFor`'s 50 ms poll makes it 100–150 ms, so a publication more than about 150 ms late is invisible; the three rejection cases assert the message but not the `ProcessCapabilityError` class; the `mcp-orphan.e2e.test.ts` comment's statement that its cases have only ever run on POSIX is historical and unverifiable; the abort case's `await spawned.promise` is unbounded (a spawn that never happens fails only as the 5 s case timeout), its closed-child `waitFor` carries no tag, and the 100/300/3,000 ms margins are bare literals. *(nit · apps/cli/src/engine/tool-host/process-close.test.ts, apps/cli/src/harness/mcp-orphan.e2e.test.ts)* -- [ ] **Native-lifetime watchdogs are fixed 5 s bounds inside 10 s cases (`LOW-1`).** *Low.* The watchdogs in +- [ ] **Scheduled → Wave 3 PR95. An `api-key` header label erases the `Bearer`/`Basic` label that follows it, so the token leaks.** *Low (security).* *(author)* `scrubSecrets('api-key:Bearer abcdefghijkl123456')` returns `api-key:[REDACTED] abcdefghijkl123456` — the `api-key` rule consumes the word `Bearer` as the header's value and the Bearer rule then has no label to anchor on. The pre-PR matcher returned `api-key:Bearer [REDACTED]`. Surfaced by the post-merge security sitting, reproduced independently; it comes from the PR's rule order, not from `a3e164b5`, and is **not fixed**. A plain `Authorization: Bearer …` is unaffected. Fix idea: let the header value skip a following `Bearer`/`Basic` label, or run those rules first. *(low · packages/llm/src/scrub-secrets.ts)* +- [ ] **Scheduled → Wave 3 PR95. URL userinfo containing a raw `?` or `#` before the first `:` (or with no `:`) is not masked.** *Ruled 2026-10-19:* mask unless the text before the `?`/`#` is host-shaped (a dotted hostname, `localhost` or a bracketed IP, with an optional `:port`) **and** an `=` assignment lies between the `?`/`#` and the authority `@`; a plain host-shaped exemption would still leave `https://my.user:1234?TOKEN@h/` open. The same predicate applies to core's twin in `packages/core/src/tools/bounding.ts` (its colonless short-handle rule and `]` delimiter stay, recorded as differences). *Low (security).* *(author, from two independent differential runs)* `https://us?er:TOKEN@h/`, `http://#TOKEN@h` and `https://my.user:1234?TOKEN@h/` pass through unchanged; the pre-PR matcher masked all of them. It follows from the deliberate host-query policy in `maskUrlUserinfo` (a raw `?`/`#` may begin public query text, e.g. `https://host?contact=a@b.c`) and the colon-first cases are pinned by tests; it is identical at `19c99968` and is not introduced by the follow-up. **A maintainer call** whether a username shaped that way is worth the false positives. Fix idea: exempt only when the text before the `?`/`#` is non-empty and host-shaped (contains `.`, a `:port`, `localhost` or a bracketed IPv6), otherwise mask. *(low · packages/llm/src/scrub-secrets.ts)* +- [ ] **Scheduled → Wave 3 PR95. Credentials in URLs of other schemes are outside the finite rule.** *Ruled 2026-10-19:* extend the finite list with postgres, postgresql, mysql, mariadb, redis, rediss, mongodb, mongodb+srv, amqp, amqps, ssh, sftp, ldap, ldaps, smtp and smtps, masking only colon-bearing userinfo for the new schemes; malformed `https:/u:p@h`, a raw `/` inside a password and invisible characters inside the scheme stay documented limits. *Low, informational.* `postgres://`, `redis://`, `mongodb(+srv)://`, `ssh://` userinfo, malformed `https:/u:p@h`, a raw `/` inside a password, and invisible characters inside the scheme are not masked, identically before and after the follow-up. The file states its shapes are finite; a database connection error is the realistic carrier, so the scheme list may deserve a decision. *(info · packages/llm/src/scrub-secrets.ts)* +- [ ] **Scheduled → Wave 3 PR95. The URL rule's split-credential scan costs 2.1–2.5× on letter-heavy input, and nothing times that loop.** *Low / nit.* With `NO_BOUNDARY` the scanner tests every ASCII letter start for the URL rule; at 8 MiB the worst case stays under about 1.1 s and linear (1–16 MiB scaling checked). A prefilter on `h`/`w`/`f` would recover most of it. The existing 8 MiB timing case contains no `@`, so it returns before the loop; a guard would use a leading `a\nhttps://u@h ` plus 8 MiB of letters, but see the load-sensitive timing item below before adding another wall-clock assertion. SonarCloud was not run on the never-matching `NO_BOUNDARY = /(?!)/`; check its next analysis. *(low · packages/llm/src/scrub-secrets.ts)* +- [ ] **Scheduled → Wave 3 PR95. Two latent invariants in `hasSplitCredential` are unchecked.** *Nit.* A boundary-bearing rule with neither `\b` nor a `joinedPattern` makes `scrubSecrets` throw a `SyntaxError` on LF/TAB input no earlier rule has redacted (the suite fails at once, so it is caught, but it would break `makeLlmError`'s no-throw claim); and `wordBlocksStart` tests `boundary.test('a')` as a proxy for "rejects every ASCII word character", true for all current boundaries. An assertion at module load would pin both. *(nit · packages/llm/src/scrub-secrets.ts)* +- [ ] **Scheduled → Wave 3 PR95. The userinfo control's joined-view half is not separately witnessed.** *Low (coverage).* *(author, from an independent run)* Mutating only `joinedPattern` of the URL rule leaves 111/111 tests green. The whitespace-stripped residual probe independently catches the class the existing cases use, so it may be largely an equivalent mutant; an added case that did not discriminate it was reverted rather than kept. No causal control pins the glued-scheme behaviour either — only ordinary tests do. *(low · tools/secrets-controls/mutations.mjs)* +- [ ] **Approved deferral (maintainer, 2026-10-19). `LOW-2` (the 1 s default `waitFor` on a real child) was examined and deliberately not changed.** *Low.* The only wait that observes a real child booting is `holdNativeToolProcess`'s marker wait, whose sole caller sits inside `drive-home.test.ts` waits (lines 633 and 637) that use the 1 s default and enclose the boot, so a longer inner bound cannot help; a retune must raise those outer waits together. No failure was reproduced (0 in dozens of loaded runs). `testing.md` also states that no readiness or case deadline is extended to pass the workspace gate. **Trigger: a recorded boot-time failure.** *(low · apps/cli/src/commands/native-forwarding.test-support.ts, apps/cli/src/home/drive-home.test.ts)* +- [ ] **Scheduled → Wave 3 17b. Join-table test nits.** *Nit.* The `process-close.test.ts` gate margin is nominally 100 ms but `waitFor`'s 50 ms poll makes it 100–150 ms, so a publication more than about 150 ms late is invisible; the three rejection cases assert the message but not the `ProcessCapabilityError` class; the `mcp-orphan.e2e.test.ts` comment's statement that its cases have only ever run on POSIX is historical and unverifiable; the abort case's `await spawned.promise` is unbounded (a spawn that never happens fails only as the 5 s case timeout), its closed-child `waitFor` carries no tag, and the 100/300/3,000 ms margins are bare literals. *(nit · apps/cli/src/engine/tool-host/process-close.test.ts, apps/cli/src/harness/mcp-orphan.e2e.test.ts)* +- [ ] **Approved deferral (maintainer, 2026-10-19). Native-lifetime watchdogs are fixed 5 s bounds inside 10 s cases (`LOW-1`).** *Low.* The watchdogs in `apps/cli/src/process/native-process-owner.test.ts`, `engine/tool-host/os-native-lifetime.test.ts`, `render/editor-native-lifetime.test.ts`, `render/fixtures/native-editor.mjs` and `tools/cli-package/smoke/native-process.test.mjs` assert that a slow-but-correct close did not trip them, so a @@ -407,36 +410,36 @@ from that review and were not re-verified by the author; items marked *(author)* does nothing, and three runs of the three vitest files (52 tests) passed at a load average of 80–96 on macOS. The reviewer could not observe the Windows root cause either. **Trigger: a Windows trace that names the bound that fired**; retune the coupled set together then. *(low · the files above)* -- [ ] **`bounds admitted-size failed header and URL preparation` is load-sensitive.** *Low.* *(author)* It allows +- [ ] **Scheduled → Wave 3 PR95. `bounds admitted-size failed header and URL preparation` is load-sensitive.** *Low.* *(author)* It allows 5 s for two 8 MiB scrubs that take about 0.75 s in total, and it failed twice with the load average near 65 (6.4 s in the run that logged the duration). Measured alternately, the pre-change code and the corrected code take the same time, so the failures were load, not the correction. Two independent reviewers saw it too (5,590 ms and 3,453 ms at load averages near 88–97) and recorded that `tools/secrets-controls/README.md` says the fixture runs without a machine-speed threshold, which this assertion contradicts; six controls run this file, so a loaded runner can turn a fixed phase red. The test guards against a quadratic regression, so any loosening must stay far below minutes. **Trigger: the next CI failure of this case.** *(low · packages/llm/src/scrub-secrets.test.ts, `SECRETS_B2_HEADER_PREPARATION`)* -- [ ] **Raw exit-code literals where `EXIT_CODES` names exist.** *Nit.* *(reviewer)* reported +- [ ] **Scheduled → Wave 3 17a (all four sites together). Raw exit-code literals where `EXIT_CODES` names exist.** *Nit.* *(reviewer)* reported `result === 5 || result === 7 || result === 8` in `apps/cli/src/chat/session-surface-lifetime.ts:54`; *(author)* the same literals occur in four places: that site, `commands/workflow-lifetime.ts:178`, `process/command-lifetime.ts:110` and `commands/drive.ts:225`. Behaviour-neutral, but it sits on the exit-code precedence that the review's red-team matrix held, so rerun the precedence tests with the change. *(nit · the four files above)* -- [ ] **CLI lifetime and API-drift nits.** *Nit, (reviewer).* `chat/session-ink-input.ts:40-42,124-131` guards only +- [ ] **Scheduled → Wave 3: `session-ink-input.ts` → 15c, `HomeDeps.exit` → 14a, the `lazy-dispatch.ts` cause-wrap → 10a. CLI lifetime and API-drift nits.** *Nit, (reviewer).* `chat/session-ink-input.ts:40-42,124-131` guards only the `readable` listener (Ink's kitty `data` path is latent because `kittyKeyboard` is unused); `home/drive-home.tsx:165-166,1067,1115-1116` keeps `HomeDeps.exit` although the owned path never calls it; `commands/lazy-dispatch.ts:49` imports the media route without the cause-wrap the sibling routes carry. *(nit · the files above)* -- [ ] **Test-evidence nits.** *Nit, (reviewer).* `process/pure-substrate-graph.test.ts:117-133` — the purity-guard +- [ ] **Scheduled → Wave 3: the purity-guard root set → 10a; the remaining test-evidence nits → 17b. Test-evidence nits.** *Nit, (reviewer).* `process/pure-substrate-graph.test.ts:117-133` — the purity-guard root set omits the ADR-0111 eager-boot files (the reviewer verified that opaque marking fixes it); three no-argument `toHaveBeenCalled()` positives outside `native-process-owner.test.ts`; `packages/llm/src/adapters/list-models-lifetime.test.ts:122` labels a fake-`fetch` case "real installed SDK"; `apps/cli/src/harness/concurrency.e2e.test.ts:434-441` uses a fixed 100 ms contention margin; `tools/cli-package/consumer-lifetime.test.mjs:54-59` pre-marks every synthetic case `passed: true` (by design; real detection is the packed-consumer CI). *(nit · the files above)* -- [ ] **Tooling and documentation nits.** *Nit, (reviewer).* Unreachable `TAR_FORMAT` branches at +- [ ] **Scheduled → Wave 3: the two tooling nits → 9a.1; the duplicate headings → 19b. Tooling and documentation nits.** *Nit, (reviewer).* *Premise corrected 2026-10-19:* the repeated headings are the 2.5.5.A–I sub-stream headings and the Wave 2/Wave 3 "Decisions presented together before implementation" pair, not Wave 2 checkpoint sub-headings. Unreachable `TAR_FORMAT` branches at `tools/cli-package/archive.mjs:126,135`; no `break` in the ancestor loop at `tools/cli-package/smoke/worker.mjs:64-65` (the outermost `node_modules` wins); and repeated heading texts in the phase register, whose later occurrences take a `-1` slug suffix. *(author)* A scan finds repeated H2–H4 headings (the 2.5.5.B–I sub-stream headings and the Wave 2 checkpoint's sub-headings among them); no link is broken today. *(nit · the files above)* -- [ ] **Informational items to keep in view.** *(reviewer)* `packages/mcp/src/manager.ts:228-236` — a comment omits +- **No action, recorded 2026-10-19 — informational items kept in view.** No change is proposed for any of them; the `git_commit` approval wording is already covered by Wave 3 Step 5's #308 disposition. *(reviewer)* `packages/mcp/src/manager.ts:228-236` — a comment omits the bare `Error` in the throwing-observer branch (behaviour is canonical); `packages/core/src/tools/bounding.ts` — the removed 4k pre-clip in `makeSummary` is required for correctness (8 MiB takes about 49 ms, about 200 ms with line feeds) and the alias DAG is `2^d` (depth 18 is 786k nodes in 404 ms) for host/embedder graphs only, never model diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 0d53c9e1..92e0f1e2 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -1,6 +1,6 @@ # Phase 2.5.5 — Hardening and Remediation -> Status: 🚧 **Open — Wave 2 Steps 1–8 and review corrections merged to main as PR #94 (2026-10-15); later waves remain open.** Live progress and execution order are canonical in +> Status: 🚧 **Open — Wave 2 merged as PR #94 (2026-10-15); Wave 3 Steps 1–8 merged as PR #95 (2026-10-18); Wave 3 Steps 9–19 and later waves remain open.** Live progress and execution order are canonical in > [../current.md](../current.md#execution-order--phase-255--phase-26-temporary); this banner does not > restate them. A full-project multi-agent code review completed > 2026-07-19 at HEAD `f88b0e8` — 377 findings, 373 surviving two-lens adversarial @@ -139,6 +139,11 @@ reached. the `coverage` job (`pnpm coverage:enforced`); no item in this phase widens the `LLMProvider` seam, adds a platform import to `packages/core`, or changes the `--json`/CI machine-output contract ([ADR-0049](../../decisions/0049-cli-machine-output-contract.md)). + *Reconciled 2026-10-19 with [exit criterion 4](#exit-criteria):* machine-output compatibility + changes are limited to the explicitly approved [ADR-0110](../../decisions/0110-cli-diagnostics-and-discovery-have-explicit-compatibility.md) + scope (diagnostics/exits, JSON discovery, aliases and consent); run-event stdout and actual + workflow-outcome contracts stay unchanged. Wave 3 Step 19 promotes `packages/core` into the + enforced coverage floor alongside `llm` and `mcp`. ## Scope @@ -194,26 +199,26 @@ reached. - ✅ **DONE (Wave 1, PR #81, 2026-07-30 — #91 · `rawPreviewFor` keeps the classification copy, `previewFor` scrubs the display copy).** **Redact tool-approval previews like `sanitizeInput()` already does.** `previewFor()` (`packages/core/src/tools/registry.ts:436`) builds the persisted, `--json`-visible `ToolActionPreview` by echoing `target.command`/`target.path` verbatim, never calling the `redactSecretShapedValue` scrub that its sibling `sanitizeInput()` (same file, line 563) applies to `toolInput` moments later in the same dispatch. A live credential in a `run_command` arg or `write_file` path is written unredacted into the durable, persisted `agent:approval_requested` event. Call the same redaction helper in `previewFor()` before the preview is built. This is the review's canonical propagation-gap example and should land first. *(S · `packages/core/src/tools/registry.ts` · #91)* - ✅ **DONE (Wave 1, PR #81, 2026-07-30 — G38 · `reservedAdmissions` ledger in `budget-governor.ts`).** **Add a reservation/admission step to `BudgetGovernor`'s pre-egress check.** `checkPreEgress` (`packages/core/src/engine/budget-governor.ts`) is a pure read against a snapshot that is only updated post-hoc after `cost:updated` lands, with no reservation/admission-control step — two or more parallel agent-node branches under `max_parallel` can each pass `checkPreEgress` against the same stale snapshot, letting the sum exceed `max_cost_microcents` even with `on_exceed:'fail'`. Add an in-flight reserved-cost ledger that is checked and updated atomically at dispatch time, and add a test combining budget + `max_parallel` (none exists today). *(L · `packages/core/src/engine/budget-governor.ts` · G38)* - ✅ **DONE (Phase 2.6.5 `CR-22`, 2026-08-27)** — closed early, out of wave order, because `W2`'s liveness item owns the same function. **Compute the remaining timeout budget on every resume instead of re-arming a fresh timer.** `#armRunTimeout` (`packages/core/src/engine/engine.ts`) re-arms a full-duration timer on every checkpoint/gate/media-job resume instead of computing remaining budget from `#elapsedMs()`, unlike the checkpoint's own `startEpochMs` and the media-job absolute-deadline pattern it should mirror — a workflow with `timeout_ms` can run far longer than authored if it pauses and resumes repeatedly. Fix `#armRunTimeout` to compute the remaining budget from elapsed wall-clock time on every re-arm. *(M · `packages/core/src/engine/engine.ts` · #75)* -- **Thread the missing `AbortSignal` into `resolvePrompt` and recheck cancellation in `human-gate.ts`.** `resolvePrompt` (`packages/core/src/engine/agent-runner.ts:868`) omits the `AbortSignal` that every other call site passes into `resolveTemplate`, so cancellation during prompt-template resolution is not observed. Separately, `human-gate.ts` (`packages/core/src/engine/node-handlers/human-gate.ts`) is the one node handler doing real async work yet has no post-resolution `ctx.signal.aborted` recheck before returning `'paused'` — a cancelled run can emit `human_gate:paused` right before `run:cancelled`. Add the signal to `resolvePrompt` and the recheck to `human-gate.ts` in one pass. *(M · `packages/core/src/engine/agent-runner.ts`, `packages/core/src/engine/node-handlers/human-gate.ts` · #76, G9)* +- ✅ **DONE ([Wave 3 Step 5](#wave-3-step-5-acceptance--2026-10-16)).** **Thread the missing `AbortSignal` into `resolvePrompt` and recheck cancellation in `human-gate.ts`.** `resolvePrompt` (`packages/core/src/engine/agent-runner.ts:868`) omits the `AbortSignal` that every other call site passes into `resolveTemplate`, so cancellation during prompt-template resolution is not observed. Separately, `human-gate.ts` (`packages/core/src/engine/node-handlers/human-gate.ts`) is the one node handler doing real async work yet has no post-resolution `ctx.signal.aborted` recheck before returning `'paused'` — a cancelled run can emit `human_gate:paused` right before `run:cancelled`. Add the signal to `resolvePrompt` and the recheck to `human-gate.ts` in one pass. *(M · `packages/core/src/engine/agent-runner.ts`, `packages/core/src/engine/node-handlers/human-gate.ts` · #76, G9)* - ✅ **DONE ([Wave 3 Step 2](#wave-3-step-2-acceptance--2026-10-16)).** **Extract a shared `messageFor` diagnostic helper for `parser.ts` and `agent-parser.ts`.** `parser.ts`'s `messageFor` has no case for Zod's `invalid_literal` code, so a `schema_version` mismatch reports only "invalid value" with no expected-value indication. `agent-parser.ts`'s `AgentParseError` carries only field paths, never reason text, so distinct error kinds report identically on the `.agent.yaml` authoring surface — a gap distinct from 2.6.B's visibility-wiring task. Extract `messageFor` into a schema-agnostic helper consumed by both `parser.ts` and `agent-parser.ts`. *(M · `packages/core/src/parser.ts`, `packages/core/src/agent-parser.ts` · #85, #86)* -- **Reconcile `system_prompt_append`'s documented interpolation with its verbatim-append implementation.** `workflow-yaml-spec.md` documents `system_prompt_append` as an interpolation target, but `agent-runner.ts` appends it verbatim by design (ADR-0038 anti-injection), while `packages/core/src/dag.ts` still wires it into real scheduling/taint-scan sites — an author using `{{ }}` there gets literal unresolved text sent to the model with zero error. Fix by either correcting the spec or adding a parse-time rejection of `{{ }}` syntax in that field. *(M · `packages/core/src/engine/agent-runner.ts`, `docs/reference/*/workflow-yaml-spec.md` · #87)* +- ✅ **DONE ([Wave 3 Step 5](#wave-3-step-5-acceptance--2026-10-16)) — documented as a literal append; Proposed ADR-0060 stays inactive.** **Reconcile `system_prompt_append`'s documented interpolation with its verbatim-append implementation.** `workflow-yaml-spec.md` documents `system_prompt_append` as an interpolation target, but `agent-runner.ts` appends it verbatim by design (ADR-0038 anti-injection), while `packages/core/src/dag.ts` still wires it into real scheduling/taint-scan sites — an author using `{{ }}` there gets literal unresolved text sent to the model with zero error. Fix by either correcting the spec or adding a parse-time rejection of `{{ }}` syntax in that field. *(M · `packages/core/src/engine/agent-runner.ts`, `docs/reference/*/workflow-yaml-spec.md` · #87)* - ✅ **DONE (Wave 1, PR #81, 2026-07-30 — G39/G47/G49 · `warningArmed` re-arm + projected `thresholdPct`).** **Re-arm `budget:warning` on continued overage and compute `thresholdPct` from the projected post-call total.** `#warningEmitted` (`packages/core/src/engine/budget-governor.ts:112`) is a one-shot lifetime boolean, so a long-lived `AgentSession`/chat process gets exactly one `budget:warning` ever and then silently proceeds no matter how far over cap spend climbs. Separately, `thresholdPct` is computed from pre-call cumulative spend rather than the projected post-call amount, so a single large call that jumps past the cap reports a near-0% warning. Fix both in the same pass: re-arm the warning on continued overage, and derive `thresholdPct` from the projected post-call total. *(M · `packages/core/src/engine/budget-governor.ts` · G39, G47, G49)* -- **Give `assertNoTaintedArgs()` an actual producer for `ctx.secretArgKeys`.** `ctx.secretArgKeys` — the runtime ADR-0029(c) taint-check input that `assertNoTaintedArgs()` (`packages/core/src/tools/registry.ts`, `packages/core/src/tools/types.ts`) correctly enforces when present — has no producer anywhere in the engine, so the check is inert everywhere, including on the `AgentSession`/chat path with no static-analysis backstop. Either wire a producer that actually populates `ctx.secretArgKeys`, or explicitly record this as a tracked, known gap in the ADR. *(M · `packages/core/src/tools/registry.ts`, `packages/core/src/engine/agent-session.ts` · #92)* -- **Cap `write_file.content`, `http_request.body`, `run_command.args`, and `git_commit.message` at a schema-level max length.** These four fields carry no length ceiling in `builtins.ts`'s Zod schemas despite heavy investment in bounding tool *output* — an oversized model-driven payload reaches dispatch and an unbounded regex redaction scan with no schema-level circuit breaker. Add a generous max-length ceiling on each field that won't break legitimate large writes. *(M · `packages/core/src/tools/builtins.ts` · #95)* -- **Rewrite `llmVisibleParams` descriptions across the built-in tool set.** No built-in tool's `llmVisibleParams` schema has per-property descriptions (byte-vs-line ranges, glob dialect, argv-vs-shell-string semantics are left to the model's guess), and several descriptions leak engineer-facing implementation vocabulary (`shell:false`) instead of actionable denial-recovery guidance. One clustered rewrite pass over `builtins.ts`'s tool descriptions, distinct from 2.6.M's new-tool scope. *(M · `packages/core/src/tools/builtins.ts` · #96, #98)* -- **Remove cycle-guard `WeakSet` entries on recursion unwind.** `redactInlineMediaForText`/`redactSecretShapedWalk` (`packages/core/src/tools/bounding.ts`) add to the shared cycle-guard `WeakSet` but never remove on the way back out of recursion, so any object referenced twice non-cyclically renders as a bogus `[cyclic]` in observability output. Remove the object from the `WeakSet` when recursion unwinds and add a regression test. *(S · `packages/core/src/tools/bounding.ts` · #94)* -- **Decide the real backpressure granularity for `RunHandle` and `SessionHandle` in one pass.** `RunHandle.whenConsumersReady()` (`packages/core/src/engine/run-handle.ts`) is only awaited once per node dispatch, never within a single node's own streaming duration, so a long token/reasoning stream against a stalled consumer buffers unboundedly. `SessionHandle.whenConsumersReady`/`whenDrained` (`packages/core/src/engine/session-handle.ts`) is implemented identically but has zero callers anywhere. Decide and implement the actual granularity needed for both handles together. *(M · `packages/core/src/engine/run-handle.ts`, `packages/core/src/engine/session-handle.ts` · #77, G48)* -- **Unescape the quoted filter-argument text `parseArg` returns.** `parseArg` (`packages/core/src/interpolation/references.ts`) returns the raw backslash-escaped text of a quoted filter argument instead of unescaping it, contradicted by its own test's comment. Fix `parseArg` to unescape the quoted argument text per its documented contract, and correct the misleading test comment. *(S · `packages/core/src/interpolation/references.ts` · #82)* +- ✅ **DISPOSED ([Wave 3 Step 3](#wave-3-step-3-acceptance--2026-10-16)) — ADR-0113 records the unsupported tool mapping honestly instead of fabricating a producer; the reproduced context leak is fixed.** **Give `assertNoTaintedArgs()` an actual producer for `ctx.secretArgKeys`.** `ctx.secretArgKeys` — the runtime ADR-0029(c) taint-check input that `assertNoTaintedArgs()` (`packages/core/src/tools/registry.ts`, `packages/core/src/tools/types.ts`) correctly enforces when present — has no producer anywhere in the engine, so the check is inert everywhere, including on the `AgentSession`/chat path with no static-analysis backstop. Either wire a producer that actually populates `ctx.secretArgKeys`, or explicitly record this as a tracked, known gap in the ADR. *(M · `packages/core/src/tools/registry.ts`, `packages/core/src/engine/agent-session.ts` · #92)* +- ✅ **DONE ([Wave 3 Step 5](#wave-3-step-5-acceptance--2026-10-16)).** **Cap `write_file.content`, `http_request.body`, `run_command.args`, and `git_commit.message` at a schema-level max length.** These four fields carry no length ceiling in `builtins.ts`'s Zod schemas despite heavy investment in bounding tool *output* — an oversized model-driven payload reaches dispatch and an unbounded regex redaction scan with no schema-level circuit breaker. Add a generous max-length ceiling on each field that won't break legitimate large writes. *(M · `packages/core/src/tools/builtins.ts` · #95)* +- ✅ **DONE ([Wave 3 Step 5](#wave-3-step-5-acceptance--2026-10-16)).** **Rewrite `llmVisibleParams` descriptions across the built-in tool set.** No built-in tool's `llmVisibleParams` schema has per-property descriptions (byte-vs-line ranges, glob dialect, argv-vs-shell-string semantics are left to the model's guess), and several descriptions leak engineer-facing implementation vocabulary (`shell:false`) instead of actionable denial-recovery guidance. One clustered rewrite pass over `builtins.ts`'s tool descriptions, distinct from 2.6.M's new-tool scope. *(M · `packages/core/src/tools/builtins.ts` · #96, #98)* +- ✅ **DONE ([Wave 3 Step 5](#wave-3-step-5-acceptance--2026-10-16)).** **Remove cycle-guard `WeakSet` entries on recursion unwind.** `redactInlineMediaForText`/`redactSecretShapedWalk` (`packages/core/src/tools/bounding.ts`) add to the shared cycle-guard `WeakSet` but never remove on the way back out of recursion, so any object referenced twice non-cyclically renders as a bogus `[cyclic]` in observability output. Remove the object from the `WeakSet` when recursion unwinds and add a regression test. *(S · `packages/core/src/tools/bounding.ts` · #94)* +- ✅ **CERTIFIED ([Wave 3 preflight](#preflight-corrections-and-already-satisfied-work)) — closed by Phase 2.6.5 CR-30 session backpressure and the W8/ADR-0087 contract; preserved, not reimplemented.** **Decide the real backpressure granularity for `RunHandle` and `SessionHandle` in one pass.** `RunHandle.whenConsumersReady()` (`packages/core/src/engine/run-handle.ts`) is only awaited once per node dispatch, never within a single node's own streaming duration, so a long token/reasoning stream against a stalled consumer buffers unboundedly. `SessionHandle.whenConsumersReady`/`whenDrained` (`packages/core/src/engine/session-handle.ts`) is implemented identically but has zero callers anywhere. Decide and implement the actual granularity needed for both handles together. *(M · `packages/core/src/engine/run-handle.ts`, `packages/core/src/engine/session-handle.ts` · #77, G48)* +- ✅ **DONE ([Wave 3 Step 5](#wave-3-step-5-acceptance--2026-10-16)).** **Unescape the quoted filter-argument text `parseArg` returns.** `parseArg` (`packages/core/src/interpolation/references.ts`) returns the raw backslash-escaped text of a quoted filter argument instead of unescaping it, contradicted by its own test's comment. Fix `parseArg` to unescape the quoted argument text per its documented contract, and correct the misleading test comment. *(S · `packages/core/src/interpolation/references.ts` · #82)* - ✅ **DONE ([Wave 3 Step 2](#wave-3-step-2-acceptance--2026-10-16)).** **Apply consistent duplicate/cross-field admission; certify existing input and session guards.** Shared leaf/full admission now enforces the approved duplicate, human-gate timeout and logical Run status matrices. Existing input defaults/enums and session compaction-role rules are certified. Historical session/workflow readers retain their former contracts; no rows are migrated or silently repaired. *(M · `packages/shared/src/{node,agent,workflow,session,run,common}.ts` · #89, #90, G18, #212, G16, G17, #210, #211)* -- **Extend `analyzeSecretTaint`'s leak-reporting loop to cover context-value sites.** `analyzeSecretTaint` (`packages/core/src/interpolation/analyze.ts`) deliberately skips context-value sites in its leak-reporting loop, so a direct `{{secrets.*}}` reference never read elsewhere is not reported at parse time — the workflow only fails later, differently-shaped, at run start. Not an actual secret leak, but a real completeness gap in the "reject before a run starts, uniformly" contract. *(S · `packages/core/src/interpolation/analyze.ts` · G22)* +- ✅ **CERTIFIED ([Wave 3 preflight](#preflight-corrections-and-already-satisfied-work)) — Wave 2's context-namespace refusal closes it.** **Extend `analyzeSecretTaint`'s leak-reporting loop to cover context-value sites.** `analyzeSecretTaint` (`packages/core/src/interpolation/analyze.ts`) deliberately skips context-value sites in its leak-reporting loop, so a direct `{{secrets.*}}` reference never read elsewhere is not reported at parse time — the workflow only fails later, differently-shaped, at run start. Not an actual secret leak, but a real completeness gap in the "reject before a run starts, uniformly" contract. *(S · `packages/core/src/interpolation/analyze.ts` · G22)* - ✅ **DONE ([Wave 3 Step 4](#wave-3-step-4-acceptance--2026-10-16)).** **Retain and validate the public event envelope without stripping full payloads.** `BaseEventSchema` shares the full run family's parsed exactly-one-correlation check; `BaseEvent` and `HumanGateEvent` remain exported. Complete payload, strict accounting, lenient opposite-key and stored forward-reader contracts are preserved. *(S · `packages/shared/src/run-event.ts` · #213)* - ✅ **DONE ([Wave 3 Step 5](#wave-3-step-5-acceptance--2026-10-16)).** **Clarify descriptive dispatch scope and retain immutable host filesystem authority.** Accepted [ADR-0105](../../decisions/0105-filesystem-roots-are-invocation-grants-not-project-authority.md) decides the host boundary; `ctx.fsScope` describes dispatch context and cannot widen the host's grant. No per-turn `FsCapability` escalation is introduced. *(S · `packages/core/src/tools/{types,registry,builtins}.ts` · #93)* -- **Correct three stale engine doc comments to match current code.** `checkpoint.ts`'s module doc (`packages/core/src/engine/checkpoint.ts`) claims a "runId+nodeId+retryCount idempotency key" bounds resumed side effects; no such key exists. `builtins.ts`'s top comment still says egress is "not wired yet"; it shipped. `workflow-yaml-spec.md` documents `workflow.version` as Semver-format-enforced; `WorkflowSpecSchema` enforces no format. One sweep correcting each. *(S · `packages/core/src/engine/checkpoint.ts`, `packages/core/src/tools/builtins.ts`, `docs/reference/*/workflow-yaml-spec.md` · #78, #97, #214)* -- **Remove the dead `RunEventBusOptions.validate` escape hatch and replace `kahnOrder`'s full re-sort with a priority structure.** `RunEventBusOptions.validate` (`packages/core/src/engine/event-bus.ts`) is documented as a hot-path escape hatch but no caller ever sets it — dead config, negligible cost today. `kahnOrder` (`packages/core/src/dag.ts`) re-sorts the entire ready-list on every pop instead of using a priority queue, also negligible at today's node counts. Both essentially-free cleanups; batch into one pass and revisit priority-queueing only if 2.6.P's subworkflows push node counts up. *(S · `packages/core/src/engine/event-bus.ts`, `packages/core/src/dag.ts` · #274, #275)* +- ✅ **DONE ([Wave 3 Step 5](#wave-3-step-5-acceptance--2026-10-16)) — #97/#214 corrected, #78 certified.** **Correct three stale engine doc comments to match current code.** `checkpoint.ts`'s module doc (`packages/core/src/engine/checkpoint.ts`) claims a "runId+nodeId+retryCount idempotency key" bounds resumed side effects; no such key exists. `builtins.ts`'s top comment still says egress is "not wired yet"; it shipped. `workflow-yaml-spec.md` documents `workflow.version` as Semver-format-enforced; `WorkflowSpecSchema` enforces no format. One sweep correcting each. *(S · `packages/core/src/engine/checkpoint.ts`, `packages/core/src/tools/builtins.ts`, `docs/reference/*/workflow-yaml-spec.md` · #78, #97, #214)* +- ✅ **DONE ([Wave 3 Step 5](#wave-3-step-5-acceptance--2026-10-16)) — #275's priority structure; #274's `validate` option is retained as an approved, documented compatibility surface.** **Remove the dead `RunEventBusOptions.validate` escape hatch and replace `kahnOrder`'s full re-sort with a priority structure.** `RunEventBusOptions.validate` (`packages/core/src/engine/event-bus.ts`) is documented as a hot-path escape hatch but no caller ever sets it — dead config, negligible cost today. `kahnOrder` (`packages/core/src/dag.ts`) re-sorts the entire ready-list on every pop instead of using a priority queue, also negligible at today's node counts. Both essentially-free cleanups; batch into one pass and revisit priority-queueing only if 2.6.P's subworkflows push node counts up. *(S · `packages/core/src/engine/event-bus.ts`, `packages/core/src/dag.ts` · #274, #275)* - ✅ **DONE ([Wave 3 Step 4](#wave-3-step-4-acceptance--2026-10-16)).** **Introduce minimal shared native error ancestry while retaining each family's contract.** The current export-scoped inventory replaces the historical 15/17 estimate: 46 roots across 37 files reach 83 module-exported production classes, with the additional unchanged private timeout descendant recorded separately. Constructors, names, discriminants, context/cause and structural/private-authority guards are preserved. *(M · `packages/shared/src/relavium-error.ts` and existing package families · #219)* -- **Chat-to-workflow export hardening: continuity caveat and non-ASCII slug fallback; certify existing tool-content exclusion.** W7/PR #90 and Accepted [ADR-0095](../../decisions/0095-what-an-agent-session-remembers-across-turns.md) structurally exclude model-issued tool arguments/results from durable session rows and exported scaffolds. The earlier raw-tool-result export claim and its proposed opt-in/redaction decision are superseded by that contract; do not add a raw-content flag. Wave 2 still adds a clear continuity/review notice and deterministic ASCII title normalization with a session-identity fallback. Retained user/assistant text, `@` attachments and `!` output can remain sensitive: this is not universal transcript masking. Certify producer→SQLite→export rejection fixtures with #115; the [Wave 2 decision package](#decisions-presented-together-before-implementation) records D14/D15. *(M · `packages/core/src/export/serializer.ts` · #80, #81, #83)* +- ✅ **CERTIFIED ([Wave 3 preflight](#preflight-corrections-and-already-satisfied-work)) — completed by W7/Wave 2.** **Chat-to-workflow export hardening: continuity caveat and non-ASCII slug fallback; certify existing tool-content exclusion.** W7/PR #90 and Accepted [ADR-0095](../../decisions/0095-what-an-agent-session-remembers-across-turns.md) structurally exclude model-issued tool arguments/results from durable session rows and exported scaffolds. The earlier raw-tool-result export claim and its proposed opt-in/redaction decision are superseded by that contract; do not add a raw-content flag. Wave 2 still adds a clear continuity/review notice and deterministic ASCII title normalization with a session-identity fallback. Retained user/assistant text, `@` attachments and `!` output can remain sensitive: this is not universal transcript masking. Certify producer→SQLite→export rejection fixtures with #115; the [Wave 2 decision package](#decisions-presented-together-before-implementation) records D14/D15. *(M · `packages/core/src/export/serializer.ts` · #80, #81, #83)* - ✅ **DONE ([Wave 3 Step 4](#wave-3-step-4-acceptance--2026-10-16)).** **Preserve typed budget-error discriminants with shared native ancestry.** `BudgetExceededError` and `BudgetPauseError` retain their `.code` fields and `instanceof` identity; the accepted D10 ruling and error-substrate increment resolve the former maintainer question. *(S · `packages/core/src/engine/budget-governor.ts` · G50)* -- **Collapse two always-co-occurring branches in `engine.ts`'s `#dispatch` retry-sleep handling.** Purely cosmetic dead-code/maintenance-trap: two branches that always co-occur, one unreachable dead weight. No behavioral bug, but real, trivially-scheduled cleanup. *(S · `packages/core/src/engine/engine.ts` · #79)* +- ✅ **DONE ([Wave 3 Step 5](#wave-3-step-5-acceptance--2026-10-16)).** **Collapse two always-co-occurring branches in `engine.ts`'s `#dispatch` retry-sleep handling.** Purely cosmetic dead-code/maintenance-trap: two branches that always co-occur, one unreachable dead weight. No behavioral bug, but real, trivially-scheduled cleanup. *(S · `packages/core/src/engine/engine.ts` · #79)* **Acceptance:** the CRITICAL `previewFor()`/approval-preview secret leak is closed and verified against a `run_command`/`write_file` fixture containing a fake credential; `BudgetGovernor` rejects a concurrent-branch overspend under `max_parallel` in a new test exercising the reservation ledger; `#armRunTimeout` computes remaining budget on every resume rather than re-arming full duration; cancel-signal propagation reaches `human-gate.ts` and `resolvePrompt`, and backpressure propagation reaches `SessionHandle`; `parser.ts` and `agent-parser.ts` share one `messageFor`-derived diagnostic helper; the approved duplicate/cross-field schema inventory is implemented or certified with its historical-reader limits in [Wave 3 Step 2](#wave-3-step-2-acceptance--2026-10-16); the cycle-guard `WeakSet` leak in `bounding.ts` is fixed with a regression test; no engine-core doc comment contradicts the code it describes; the budget-error `.code` ruling is resolved by D10 and [Wave 3 Step 4](#wave-3-step-4-acceptance--2026-10-16), while the export privacy/warning scope is resolved by the accepted D14/D15 Wave 2 contract. These are resolved decision gates, not current blockers; whole sub-stream A acceptance remains Wave 3 Step 19. @@ -309,20 +314,20 @@ This workstream is `apps/cli`'s outermost layer — the process/runtime host eve **Tasks:** -- **Close the five remaining gaps in the CLI's outermost crash/signal safety net.** `processIo()` (`apps/cli/src/process/io.ts`) has no EPIPE handling, so a `--json` consumer that closes its reader early dumps a raw stack trace instead of a structured envelope. `buildProgram()` in `apps/cli/src/run.ts` is the one synchronous step inside `run()` not wrapped in the standard error path, despite `run()`'s own JSDoc promising it never rejects. There is zero `process.on('unhandledRejection'|'uncaughtException', ...)` registration anywhere in production `apps/cli/src` — an async error escaping `run()`'s own promise chain falls straight to Node's default handler, and `process.on('exit')` never reaps detached `run_command` child process groups or closes DB/MCP connections on a crash. The last-resort crash message in `apps/cli/src/index.ts` unconditionally writes plain text to stderr with no `--json` check, the sole exception to the CLI's otherwise-universal structured-envelope convention. Fix all five in `index.ts`/`run.ts`/`process/io.ts` together: add EPIPE handling, wrap `buildProgram()`, register the missing global handlers with child-process reaping and DB/MCP close-on-exit, and make the last-resort message honor `--json`. *(L · apps/cli/src/index.ts, apps/cli/src/run.ts, apps/cli/src/process/io.ts · #40, #41, #42, #224, #269)* **Source correction — 2026-10-15:** the recoverable rejection net already exists and keeps alive; preserve it. Proposed ADR-0110 adds fatal/early-import/output handling through registered owners. Do not close DB/MCP from exit without actual owner ACK or claim universal descendant containment; see the revised preflight and canonical proposed teardown scope. +- ✅ **DONE ([Wave 3 Step 8](#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17)).** **Close the five remaining gaps in the CLI's outermost crash/signal safety net.** `processIo()` (`apps/cli/src/process/io.ts`) has no EPIPE handling, so a `--json` consumer that closes its reader early dumps a raw stack trace instead of a structured envelope. `buildProgram()` in `apps/cli/src/run.ts` is the one synchronous step inside `run()` not wrapped in the standard error path, despite `run()`'s own JSDoc promising it never rejects. There is zero `process.on('unhandledRejection'|'uncaughtException', ...)` registration anywhere in production `apps/cli/src` — an async error escaping `run()`'s own promise chain falls straight to Node's default handler, and `process.on('exit')` never reaps detached `run_command` child process groups or closes DB/MCP connections on a crash. The last-resort crash message in `apps/cli/src/index.ts` unconditionally writes plain text to stderr with no `--json` check, the sole exception to the CLI's otherwise-universal structured-envelope convention. Fix all five in `index.ts`/`run.ts`/`process/io.ts` together: add EPIPE handling, wrap `buildProgram()`, register the missing global handlers with child-process reaping and DB/MCP close-on-exit, and make the last-resort message honor `--json`. *(L · apps/cli/src/index.ts, apps/cli/src/run.ts, apps/cli/src/process/io.ts · #40, #41, #42, #224, #269)* **Source correction — 2026-10-15:** the recoverable rejection net already exists and keeps alive; preserve it. Proposed ADR-0110 adds fatal/early-import/output handling through registered owners. Do not close DB/MCP from exit without actual owner ACK or claim universal descendant containment; see the revised preflight and canonical proposed teardown scope. - ✅ **DONE — Wave 3 Step 1, 2026-10-16** ([acceptance](#wave-3-step-1-acceptance--2026-10-16)). **Carry `const exhaustive: never` from `engine.ts` to the three sibling switches it was never propagated to, two of which drop live data.** `apps/cli/src/chat/persister.ts`'s event switch — the sole writer of `agent_sessions`/`session_costs` — silently drops any unhandled event kind via a bare `default`, a real money/data risk. Four picker key-routing switches in `apps/cli/src/render/tui/chat-ink.tsx` and `apps/cli/src/render/tui/home-controller.ts` lack the same guard on the reseat surface that already caused a data-loss-class bug. `apps/cli/src/render/tui/run-view-model.ts`'s `reduceRunEvent` silently drops `agent:reasoning` with no state field to hold it, so `relavium run` never shows reasoning that `chat` already renders — the exact bug class the guard in `apps/cli/src/chat/session-host.ts`'s `engine.ts`-derived pattern was invented to prevent, recurring on a different driver. Apply `const exhaustive: never` uniformly across all six sites. *(M · apps/cli/src/chat/persister.ts · #300, #301, G3)* - **Centralize `openLocalDb` wrap-and-redact instead of leaving it well-wrapped at only 3 of 8+ call sites.** On the `run`/`chat`/`gate` paths the well-wrapped open happens AFTER an earlier unwrapped one, making the clean `CliError` message largely unreachable in practice. Separately, `packages/db/src/client.ts` interpolates a raw absolute path into its error with no redaction backstop, unlike `packages/llm`'s `scrubSecrets`. Move the wrap-and-redact logic inside `openLocalDb` itself so every caller gets it for free instead of re-implementing it. *(M · packages/db/src/client.ts · #278, #280)* - **Wire `agent-run.ts`'s missing SIGINT handler and make chat's existing one actually abort the session.** `apps/cli/src/commands/agent-run.ts` registers zero SIGINT handler, so Ctrl-C orphans spawned MCP child processes. `drivePlain`/`driveJson`'s SIGINT handlers (chat driver) only close readline and never call `session.abort()`, letting the billed generation run to completion. Register a handler in `agent-run.ts` mirroring `driveRun`'s teardown, and make the chat handlers call `abort()`. *(M · apps/cli/src/commands/agent-run.ts · #21, #23)* - **Wrap `gate.ts`'s two idempotent-resume success paths in the same `--json` branch every other command uses.** Both write plain human text to stdout unconditionally, breaking the NDJSON contract for scripted callers, and neither path is tested. No dependency on the 2.6.K resume-core extraction — fix in place. *(S · apps/cli/src/commands/gate.ts · #20)* - **Fix `/doctor`'s hardcoded probe profile, which misreports the exact thing it exists to diagnose.** `apps/cli/src/chat/doctor.ts` probes tool-host wiring with a hardcoded profile instead of the session's actual one, so it always misreports `egress`/`os` capability as unwired regardless of the real config. Probe with the session's actual profile. *(S · apps/cli/src/chat/doctor.ts · G43)* - **Implement `--help --json`'s promised `COMMAND_MANIFEST` serialization, or correct the docs that claim it exists.** Doc comments in `apps/cli/src/commands/manifest.ts` and user-facing docs both claim `--help --json` emits `COMMAND_MANIFEST` as JSON; no code path serializes it, confirmed by a passing test that asserts help stays human text. Either implement the missing serialization or amend the docs to match reality. *(M · apps/cli/src/commands/manifest.ts · #13)* -- **Defer the CLI's eager `driveHome` import and unconditional `seedCatalog()` call behind command-need, closing the ~400ms `--help`/`--version` tax.** `apps/cli/src/run.ts`'s static import of `driveHome` pulls in its full transitive graph — ink, react, the home controller, the DB layer, the native keychain binding — on every process start, even for `--help`/`--version`. `seedCatalog()` (`apps/cli/src/engine/catalog-refresh.ts`) does an unconditional sync read/parse/validate of the cached model catalog before checking whether pricing data is even needed. Measured ~390-410ms wall time for `--version` against ~30ms bare Node startup; a separate measurement centered on `dispatch.ts` confirmed the same root cause independently. Gate both behind a dynamic import keyed on actual command need; likely requires enabling `splitting: true` in `apps/cli/tsup.config.ts`, which currently emits a single bundle file. *(L · apps/cli/src/run.ts · #43, #44, #271)* +- ✅ **DONE ([Wave 3 Step 7](#wave-3-step-7b-and-whole-step-7-acceptance--2026-10-16)) — lazy packaged runtime under ADR-0111.** **Defer the CLI's eager `driveHome` import and unconditional `seedCatalog()` call behind command-need, closing the ~400ms `--help`/`--version` tax.** `apps/cli/src/run.ts`'s static import of `driveHome` pulls in its full transitive graph — ink, react, the home controller, the DB layer, the native keychain binding — on every process start, even for `--help`/`--version`. `seedCatalog()` (`apps/cli/src/engine/catalog-refresh.ts`) does an unconditional sync read/parse/validate of the cached model catalog before checking whether pricing data is even needed. Measured ~390-410ms wall time for `--version` against ~30ms bare Node startup; a separate measurement centered on `dispatch.ts` confirmed the same root cause independently. Gate both behind a dynamic import keyed on actual command need; likely requires enabling `splitting: true` in `apps/cli/tsup.config.ts`, which currently emits a single bundle file. *(L · apps/cli/src/run.ts · #43, #44, #271)* - ✅ **DONE — Wave 3 Step 1, 2026-10-16** ([acceptance](#wave-3-step-1-acceptance--2026-10-16)). **Extend `authoring.ts`'s cwd-relative `displayPath` error-path discipline to `resolve.ts`, `catalog.ts`, and `chat/export.ts`.** `apps/cli/src/authoring/authoring.ts`'s `readAuthoredFile` already renders filesystem errors against a cwd-relative `displayPath` instead of the raw absolute path; `apps/cli/src/workflows/resolve.ts` and `apps/cli/src/workflows/catalog.ts` never adopted it, and neither did the filesystem `CliError`s in `apps/cli/src/commands/chat-export.ts`. Apply `displayPath` at all three remaining sites so no command leaks the user's absolute home-directory path into an error message. *(M · apps/cli/src/workflows/resolve.ts · #4, G11)* - ✅ **DONE — Wave 3 Step 1, 2026-10-16** ([acceptance](#wave-3-step-1-acceptance--2026-10-16)). **Fix `resolve.ts`'s 1x UTF-8 byte multiplier, which undersizes the load cap for non-ASCII workflow files.** `apps/cli/src/workflows/resolve.ts` compares `stats.size` directly against `MAX_SOURCE_CHARS` (a 1x multiplier); `authoring.ts` and `catalog.ts` both correctly use `MAX_SOURCE_CHARS * 4` to account for UTF-8's up-to-4-bytes-per-char worst case. On the `relavium run` load path this means a dense multibyte YAML file well under the real char cap can be wrongly rejected as oversized. Apply the same `* 4` multiplier used by its two siblings. *(S · apps/cli/src/workflows/resolve.ts · #3)* - ✅ **DONE — Wave 3 Step 1, 2026-10-16** ([acceptance](#wave-3-step-1-acceptance--2026-10-16)). **Extend `chat.ts`'s `closeQuietly()` helper to the four sibling commands still using an unguarded `.close()`.** `apps/cli/src/commands/chat.ts` wraps every store-close in `closeQuietly()` so a close-time throw can never overwrite a successful exit code; `apps/cli/src/commands/run.ts`, `apps/cli/src/commands/gate.ts`, `apps/cli/src/commands/chat-export.ts`, and `apps/cli/src/commands/chat-list.ts` all still call `.close()`/`opened.close()` unguarded. Extract the helper to a shared location and import it into all four. *(S · apps/cli/src/commands/chat.ts · #22)* - **Close `agent run`'s two UX-parity gaps against its sibling drivers in one pass.** `apps/cli/src/commands/agent-run.ts` blocks silently on a TTY with no piped stdin and prints no banner, unlike `chat`/`create` which both print one. `makePlainPrinter` (shared with `apps/cli/src/commands/chat.ts`'s plain-printer path) has no `cost:updated` case or final cost line for `agent run`'s non-JSON output, unlike `run`'s TUI/plain renderer. Both are the same shape of gap — `agent run` is the newest, least-mature driver — so add the stdin-TTY banner `create` already has and the cost line `run` already prints in the same change. *(S · apps/cli/src/commands/agent-run.ts · #24, #26)* -- **Add a dedicated exit code for unclassified internal faults, or amend the docs that call exit code 1 "workflow failed."** Exit code 1 is documented as meaning a failed workflow but is silently reused for any unclassified internal fault from any command, so a script branching on exit code cannot distinguish "your workflow failed" from "the CLI itself broke." Add a dedicated code or correct the documented contract. *(S · apps/cli/src/index.ts · G35)* -- **Resolve `--quiet`/`--verbose`/`RELAVIUM_DEBUG` into one coherent diagnostic-flag system and point the CLI's generic error fallbacks at it.** `--quiet` has exactly one consumer, and that consumer hardcodes the value away — a complete no-op. `--verbose` only gates `err.stack` on a top-level crash and has zero effect during a live session. `RELAVIUM_DEBUG` is undocumented and is a second, inconsistently-named escape hatch alongside `--verbose`. Both of the CLI's generic "internal error" fallback messages never mention `--verbose` or `RELAVIUM_DEBUG` as a way to get more detail — the same incoherence, seen from the user's side of the same broken flag system. Wire `--quiet` to actually suppress output, extend `--verbose` to gate live-session diagnostics, fold `RELAVIUM_DEBUG` into `--verbose`'s precedence chain, and append the resulting hint to both generic fallback messages in the same change; the fallback-message fix cannot be written correctly until the flag relationship is settled. **Approved 2026-10-16 in ADR-0110:** quiet+verbose conflict, explicit flags over exact DEBUG=1, independent JSON representation and preserved requested data/safety categories. Implementation remains staged. *(M · apps/cli/src/index.ts · #45, #264, #265, #268)* +- ✅ **DONE ([Wave 3 Step 8](#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17)) — `cli_internal`/`output_unavailable` exit 9 under ADR-0110, with Step 6's contracts.** **Add a dedicated exit code for unclassified internal faults, or amend the docs that call exit code 1 "workflow failed."** Exit code 1 is documented as meaning a failed workflow but is silently reused for any unclassified internal fault from any command, so a script branching on exit code cannot distinguish "your workflow failed" from "the CLI itself broke." Add a dedicated code or correct the documented contract. *(S · apps/cli/src/index.ts · G35)* +- **Resolve `--quiet`/`--verbose`/`RELAVIUM_DEBUG` into one coherent diagnostic-flag system and point the CLI's generic error fallbacks at it.** `--quiet` has exactly one consumer, and that consumer hardcodes the value away — a complete no-op. `--verbose` only gates `err.stack` on a top-level crash and has zero effect during a live session. `RELAVIUM_DEBUG` is undocumented and is a second, inconsistently-named escape hatch alongside `--verbose`. Both of the CLI's generic "internal error" fallback messages never mention `--verbose` or `RELAVIUM_DEBUG` as a way to get more detail — the same incoherence, seen from the user's side of the same broken flag system. Wire `--quiet` to actually suppress output, extend `--verbose` to gate live-session diagnostics, fold `RELAVIUM_DEBUG` into `--verbose`'s precedence chain, and append the resulting hint to both generic fallback messages in the same change; the fallback-message fix cannot be written correctly until the flag relationship is settled. **Approved 2026-10-16 in ADR-0110:** quiet+verbose conflict, explicit flags over exact DEBUG=1, independent JSON representation and preserved requested data/safety categories. Implementation remains staged. *Note 2026-10-19:* the [ADR-0110 2026-10-18 clarification](../../decisions/0110-cli-diagnostics-and-discovery-have-explicit-compatibility.md#fixed-infrastructure-diagnostic-clarification--2026-10-18) supersedes this bullet's "append the resulting hint" instruction: infrastructure and unclassifiable faults keep one fixed diagnostic at every verbosity and their message does **not** suggest `--verbose` or `RELAVIUM_DEBUG` ([commands.md](../../reference/cli/commands.md#diagnostic-policy)). #45 closes on that disposition in Wave 3 Step 10a. *(M · apps/cli/src/index.ts · #45, #264, #265, #268)* - **Certify durable-only logs (G15; source correction — 2026-10-15).** The earlier token-firehose premise is refuted: shipping token/reasoning emission is live-only and logs reads durable rows. No coalescing decision or new persistence is required. Preserve actual replay with a real engine/store/log regression and JSON sequence-gap controls; the [reopened preflight](#maintainer-review-reopened-the-proposal--2026-10-15) records the evidence. *(M · apps/cli/src/commands/logs.ts · G15)* - **Attach `GLOBAL_OPTIONS_HELP` to every subcommand's `--help`, not just the root program's.** `apps/cli/src/program.ts` defines `GLOBAL_OPTIONS_HELP` (documenting the position-independent flags `commander` itself doesn't parse, like `--json`/`--cwd`/`--verbose`) and attaches it only to the root program; `commander` does not propagate it to subcommand help output, so `relavium run --help` never shows it. Attach the same help text to each subcommand, or add an explicit pointer to `relavium --help`. *(S · apps/cli/src/program.ts · #17)* - **Add a `--force` gate or interactive confirmation to `provider remove-key`, the manifest's sole `effect: 'destructive'` command.** `apps/cli/src/commands/provider.ts`'s `remove-key` action has no confirmation gate at all, unlike the create/import/export family, which all require explicit confirmation for destructive effects. Add the same pattern here. *(S · apps/cli/src/commands/provider.ts · #19)* @@ -380,10 +385,10 @@ Cross-cutting code-quality debt with no owning subsystem, concentrated in `packa - **Expand tabs before measuring transcript width.** `stripTerminalControls` (`apps/cli/src/render/sanitize.ts`) deliberately preserves `\t` in transcript body text, but `displayWidth`/`graphemeWidth` (`apps/cli/src/render/tui/viewport.ts`) delegate to `string-width`, which measures any control character — including tab — as 0-width, while a real terminal expands a literal tab to the next 8-column stop. This breaks the viewport module's own documented invariant ("1 DisplayLine == 1 terminal row"): scroll offsets and mouse row→line mapping drift whenever a tab reaches the transcript (shell/tool output, TSV data, a tab-indented code block a model emits). Expand tabs to spaces at the same display boundary `sanitizeInline` already uses for short fields, or give `wrapLogicalLine`/`displayWidth` explicit tab-stop-aware accounting, and add a `viewport.test.ts` case pinning the chosen behavior. *(M · apps/cli/src/render/tui/viewport.ts · #65)* - **Replace four hand-rolled money/token reimplementations with the canonical helpers they already sit next to.** `capUsd` (`apps/cli/src/chat/effort-notice.ts`) hand-rolls the same ÷1e8 microcent→USD conversion `format.ts`'s `formatCostUsd`/`formatCostShort` already canonicalize — against `format.ts`'s own explicit "never a second hand-rolled divisor" comment (#236). `TOKENS_PER_MTOK` is declared independently in both `packages/llm/src/budget-estimator.ts` and `packages/llm/src/cost-tracker.ts`, despite `budget-estimator.ts` already importing other symbols from `cost-tracker.js` (#237). `mention.ts` (`apps/cli/src/render/tui/mention.ts`) hardcodes its own `Math.ceil(bytes / 4)` for the `@`-mention token warning instead of importing `CHARS_PER_TOKEN` from `packages/llm/src/adapters/shared.ts` (#242). `budgetWarningText` (`apps/cli/src/commands/chat.ts`) interpolates raw microcents directly ("~80% of the 500000000µ¢ cap reached") instead of calling `formatCostUsd`/`formatCostShort` — the one cost-facing string in the CLI that doesn't (#307). One mechanical replace-with-import pass, no new logic, closes all four. *(S · apps/cli/src/chat/effort-notice.ts, packages/llm/src/budget-estimator.ts, apps/cli/src/render/tui/mention.ts, apps/cli/src/commands/chat.ts · #236, #237, #242, #307)* - **Split the `Capability` vocabulary collision and unify it with `Port`.** `packages/llm/src/capabilities.ts`'s bare `Capability = keyof CapabilityFlags` (a model feature flag) collides with `packages/core/src/tools/types.ts`'s `FsCapability`/`ProcessCapability`/`EgressCapability`/`OsCapability`/`McpCapability` (host-injected ToolHost-seam implementations) — the same root word for two unrelated architectural roles, hurting greppability on exactly the seam a security review audits (#217). The identical host-injected pattern is separately spelled `Port` elsewhere (`HomeModelsPort` in `apps/cli/src/render/tui/home-controller.ts`, `SuspendPort`, `MediaWritePort`, `MediaReferencePort`), so `apps/cli` imports both spellings for one concept in sibling files (#222). Rename `packages/llm`'s bare `Capability` alias to `ModelFeature`/`FeatureFlag`, leave `core`'s `*Capability` family (already matching ADR-0037/ADR-0041's ToolHost vocabulary) as sole owner of the word, and settle on `Port` as the one name for "host-injected dependency interface," recording the convention in `docs/standards/architectural-principles.md`. *(S · packages/llm/src/capabilities.ts, packages/core/src/tools/types.ts · #217, #222)* -- **Pick one spelling each for `ok`/`success` and `model`/`modelId`.** `packages/core` uses `{ ok: true | false }` at 49 call sites across the package (`agent-runner.ts`, `agent-session.ts`), but `tools/registry.ts` and `engine/agent-turn.ts` use `success: true | false` for the identical binary-outcome shape (#216). Within `packages/llm`'s catalog/pricing layer, lookups split roughly 50/50 between `modelId` (`pricing.ts`'s `catalogPricing`, `cost-tracker.ts`) and `model` (`pricing.ts`'s `contextWindowForModel`, `output-cap.ts`, `adapters/shared.ts`) for the same lookup-key role. Rename the two `success` sites to `ok` (or document the wire-schema exception for `ToolResultEventData.success`), and standardize on `modelId` for every catalog/pricing-lookup parameter, reserving `model` for signatures that literally forward `LlmRequest.model`. *(S · packages/core/src/tools/registry.ts, packages/core/src/engine/agent-turn.ts, packages/llm/src/pricing.ts · #216, #221)* +- ✅ **DONE ([Wave 3 Step 5](#wave-3-step-5-acceptance--2026-10-16)) — #221; #216's canonical event `success` field is retained per the preflight disposition.** **Pick one spelling each for `ok`/`success` and `model`/`modelId`.** `packages/core` uses `{ ok: true | false }` at 49 call sites across the package (`agent-runner.ts`, `agent-session.ts`), but `tools/registry.ts` and `engine/agent-turn.ts` use `success: true | false` for the identical binary-outcome shape (#216). Within `packages/llm`'s catalog/pricing layer, lookups split roughly 50/50 between `modelId` (`pricing.ts`'s `catalogPricing`, `cost-tracker.ts`) and `model` (`pricing.ts`'s `contextWindowForModel`, `output-cap.ts`, `adapters/shared.ts`) for the same lookup-key role. Rename the two `success` sites to `ok` (or document the wire-schema exception for `ToolResultEventData.success`), and standardize on `modelId` for every catalog/pricing-lookup parameter, reserving `model` for signatures that literally forward `LlmRequest.model`. *(S · packages/core/src/tools/registry.ts, packages/core/src/engine/agent-turn.ts, packages/llm/src/pricing.ts · #216, #221)* - **Rename `swapAgentModel` to match the dominant "reseat" vocabulary.** ADR-0059 ("CLI mid-session model reseat") and the surrounding codebase use `reseat*` identifiers pervasively (100+ occurrences in `apps/cli/src/commands/chat.ts` alone: `reseatChat`, `ReseatTarget`, `createReseatRebuild`, `finalizeReseatOutcome`). Yet the actual mutation function performing the model rebind — `swapAgentModel` (`apps/cli/src/chat/session-host.ts`), whose own docstring calls it a "model SWITCH" — and the dispatcher `resolveSwapRebuild` (`chat.ts`), which branches on `outcome.kind === 'reseat'`, both use a third word for the same operation, so a grep for "reseat" against ADR-0059's implementation misses it entirely. 2.6.C is closed, so this is a maintenance rename against shipped code, not new scope: rename `swapAgentModel` → `reseatAgentModel`, keeping "swap" only as the deliberately broader umbrella term where `resolveSwapRebuild` genuinely branches on two outcome kinds. *(S · apps/cli/src/chat/session-host.ts · #218)* - **Delete three knip-confirmed dead exports and their false justifying comments.** The `@relavium/llm/adapters` subpath export (`packages/llm/src/adapters/index.ts`) has zero real consumers — `packages/llm/src/providers.ts` imports the three adapter factories directly from their own files, bypassing the barrel entirely (#239). `isDispatchableId` (`apps/cli/src/commands/dispatch.ts`) is exported with a doc comment claiming "a surface calls this before dispatch," but no call site exists anywhere in the repo (#240). `FRAME_MS`'s re-export from `apps/cli/src/render/tui/ink-renderer.ts` carries a comment claiming "existing importers" use it, but every real consumer (`drive-home.tsx`, `chat-ink.tsx`, `synchronized-output.test.tsx`) imports it directly from `tui-constants.js` instead (#241). Delete each dead export/re-export and its stale comment, or, for the `/adapters` subpath, replace the comment with a note on which future consumer it is reserved for if kept deliberately. *(S · packages/llm/src/adapters/index.ts, apps/cli/src/commands/dispatch.ts, apps/cli/src/render/tui/ink-renderer.ts · #239, #240, #241)* -- **Pin every identifier-sorting `localeCompare` call to break the machine-locale dependency.** `models refresh --json`'s provider sort (`apps/cli/src/engine/model-refresh.ts`) calls `a.provider.localeCompare(b.provider)` with no locale argument, so the comparison resolves against the invoking machine's ambient `Intl` locale (e.g. `LANG`) rather than anything Relavium controls — breaking the ADR-0049 byte-identical `--json` promise, since e.g. Turkish collation reorders relative to `en_US` (#258). The same unpinned pattern recurs at 7+ more sites sorting closed-set ASCII identifiers: `apps/cli/src/engine/mcp-servers.ts` (env var names, tool allowlist), `apps/cli/src/workflows/catalog.ts` (workflow/agent slug order feeding both the Home browser and `relavium list`), `apps/cli/src/commands/list.ts`, `apps/cli/src/render/tui/mention.ts`, and `packages/core/src/tools/registry.ts`/`errors.ts` (`UnknownToolError`'s message, which can propagate into a `node:failed` RunEvent's `outputSummary` and thus into the NDJSON stream too) (#259). One shared `compareIdentifiers(a, b)` helper — a plain ASCII ordinal comparison, matching the one site that already gets this right (`packages/llm/src/model-catalog.ts`'s `localeCompare(y.displayName, 'en')`) — closes the whole sweep in one pass. *(M · apps/cli/src/engine/model-refresh.ts, apps/cli/src/engine/mcp-servers.ts, apps/cli/src/workflows/catalog.ts, packages/core/src/tools/registry.ts · #258, #259)* +- ✅ **DONE ([Wave 3 Step 5](#wave-3-step-5-acceptance--2026-10-16)).** **Pin every identifier-sorting `localeCompare` call to break the machine-locale dependency.** `models refresh --json`'s provider sort (`apps/cli/src/engine/model-refresh.ts`) calls `a.provider.localeCompare(b.provider)` with no locale argument, so the comparison resolves against the invoking machine's ambient `Intl` locale (e.g. `LANG`) rather than anything Relavium controls — breaking the ADR-0049 byte-identical `--json` promise, since e.g. Turkish collation reorders relative to `en_US` (#258). The same unpinned pattern recurs at 7+ more sites sorting closed-set ASCII identifiers: `apps/cli/src/engine/mcp-servers.ts` (env var names, tool allowlist), `apps/cli/src/workflows/catalog.ts` (workflow/agent slug order feeding both the Home browser and `relavium list`), `apps/cli/src/commands/list.ts`, `apps/cli/src/render/tui/mention.ts`, and `packages/core/src/tools/registry.ts`/`errors.ts` (`UnknownToolError`'s message, which can propagate into a `node:failed` RunEvent's `outputSummary` and thus into the NDJSON stream too) (#259). One shared `compareIdentifiers(a, b)` helper — a plain ASCII ordinal comparison, matching the one site that already gets this right (`packages/llm/src/model-catalog.ts`'s `localeCompare(y.displayName, 'en')`) — closes the whole sweep in one pass. *(M · apps/cli/src/engine/model-refresh.ts, apps/cli/src/engine/mcp-servers.ts, apps/cli/src/workflows/catalog.ts, packages/core/src/tools/registry.ts · #258, #259)* - **Estimate attachment token warnings from injected content, not raw file size.** `fileAttachmentWarning` (`apps/cli/src/render/tui/attachments.ts`) documents its estimate as computed "from the ACTUALLY-INJECTED (bounded) length — NOT the raw file size," but the implementation always derives `injectedTokens` from the `sizeBytes` parameter, never from `content.length`. For ASCII content the two values are numerically equal, masking the bug, but for any multi-byte UTF-8 content (non-Latin scripts, emoji, i18n fixtures) `sizeBytes` exceeds `content.length`, so the non-truncated-branch warning reports an inflated token estimate against the function's own documented contract. Switch to `Math.min(content.length, INJECT_MAX_CHARS)`, and extend the existing test with a `content`/`sizeBytes` pair that diverge, since the current fixture passes identical values for both and cannot catch this. *(S · apps/cli/src/render/tui/attachments.ts · G54)* - **Harden two structurally-checked type escapes that rely on control flow, not the compiler.** `catalog-metadata.ts` (`apps/cli/src/engine/catalog-metadata.ts`) casts Zod-validated catalog data to `CatalogPriceTier[]`/`ReasoningControls`/`RequestCapabilities` via `as X` on a money/capability-adjacent boundary feeding the ADR-0028 cost cap — TS rejects a structurally-incompatible cast, but not a future schema change that stays structurally assignable while becoming semantically wrong (#303). `schema-compiler.ts` (`packages/mcp/src/schema-compiler.ts`) uses non-null assertions (`built[0]!`, `literals[0]!`) that are correct today only because an adjacent `if` guards the invariant, not because `noUncheckedIndexedAccess` enforces it — a future reorder of that guard would silently reintroduce a runtime crash with no compiler signal (#304). Add a round-trip fixture test pinning the catalog schema-to-interface mapping for the former; destructure-and-check instead of asserting for the cheap cases of the latter, commenting the rest. *(S · apps/cli/src/engine/catalog-metadata.ts, packages/mcp/src/schema-compiler.ts · #303, #304)* - **Bound `SGR_MOUSE`'s digit groups to match the file's own defensive posture.** `SGR_MOUSE = /^\x1b?\[<(\d+);(\d+);(\d+)([Mm])$/` (`apps/cli/src/render/tui/mouse.ts`) accepts an unbounded digit run in each field on the complete-report path, while the partial-fragment path in the same file caps at `MAX_PARTIAL_LENGTH` (24) specifically "so a malformed stream cannot grow a buffer." Not currently exploitable — `Number()` on an oversized digit string saturates rather than throwing, and a real terminal only ever emits 1-5 digit fields — but it is an inconsistency against the file's own stated bounding discipline. Cap each digit group to `\d{1,5}`, matching the fragment path's defensive intent. *(S · apps/cli/src/render/tui/mouse.ts · G55)* @@ -969,6 +974,13 @@ The review report itself is not tracked in git (it lives under the gitignored `d #### Routed into the active Phase 2.6 — these land in that phase's own workstreams, not in this file +> **Pull-forward note — 2026-10-19:** `G30` (`http_request` advertised in chat with no allowlist path) is no +> longer 2.6.M work. The approved Wave 3 plan pulled it forward to +> [execution row 13](#proposed-implementation-increments) under Accepted +> [ADR-0112](../../decisions/0112-chat-network-grants-and-onboarding-state-are-operator-owned.md): a global-only +> `[chat_http].allowed_domains` with live narrowing authority, not coupled to credential-resolver or `web_search` +> activation. The generated row below keeps its original routing as history. + | Finding | Severity | Location | Issue | Lands in | Work item | |---|---|---|---|---|---| | `#88` | `high` | `packages/core/src/validate-catalog.ts:65` | D15 catalog load-check only fires on a node-level model override — dead for the normal pattern | `2.6.A` | nodeCatalogIssue only reads a node-level model override, never the resolved agent's own model… | @@ -3629,6 +3641,11 @@ the token heuristic is now in LLM `request-estimator`, not an adapter module. #### Decisions presented together before implementation +> **Approved 2026-10-16** — see [Wave 3 maintainer approval and implementation checkpoint](#wave-3-maintainer-approval-and-implementation-checkpoint--2026-10-16). +> Steps 1–8 merged as PR #95; the 2026-10-19 rulings for Steps 9–19 are in +> [Wave 3 Steps 9–19 preflight review and maintainer rulings](#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19). +> Live progress is canonical in [current.md](../current.md#wave-3--clear-the-ground). + All entries below are **proposed**, not maintainer rulings. Approval covers the concrete public/API/secrets plan as well as the new ADRs. No proposed choice silently changes a currently Accepted ADR or a canonical reference before approval. @@ -3655,6 +3672,12 @@ No credential values or request captures belong in tracked evidence. #### Proposed implementation increments +> **Approved 2026-10-16** ([checkpoint](#wave-3-maintainer-approval-and-implementation-checkpoint--2026-10-16)). +> Rows 7a–8d, 9a–9d and every `Recorded split` / `Recorded plan amendment` row below were added later with the +> maintainer's approval; the 2026-10-19 amendment is recorded in +> [Wave 3 Steps 9–19 preflight review and maintainer rulings](#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19). +> Steps 1–8 merged as PR #95; live progress is canonical in [current.md](../current.md#wave-3--clear-the-ground). + | Step | Scope, finding IDs and principal verification | |---|---| | 1 | **Serialized multi-file queue first:** shared close handling #22 (four named siblings plus newer owned journal sites) → safe source labels #4/G11 plus #3's UTF-8 source admission parity → exhaustive session/picker/run switches #300/#301/G3 and actual reasoning rendering. Preserve primary outcomes and host-safe-close ownership. For #3, use the sibling 4× byte precheck plus authoritative parser character bound; test a dense multibyte source above the old byte cap but below the character cap, ASCII character overflow and true byte overflow. Add native error and mounted output regressions. | @@ -3671,21 +3694,46 @@ No credential values or request captures belong in tracked evidence. | 8b | **Recorded split — creation-owned native processes:** tool POSIX group versus MCP PID, actual close/retirement, verified absolute Windows teardown admission and helper lifetime; connect OS/editor helpers and replace independent reapers without claiming full descendant containment. | | 8c | **Recorded split — workflow and render integration:** run/gate/budget/driver owners, real return/throw health priority, cancellation, terminal/job-control leases and force-quit routing. No timer certifies DB close or receipt settlement. | | 8d | **Recorded split — sessions, terminal and activation:** fresh/resumed/one-shot/Home/pending/replaced session lifetimes, retained publication failure and shared actual cleanup promises; connect caught slash/rebuild faults, restore terminal custody before diagnosis, then activate early coordinator/background/sink integration. Actual native/platform/packed controls and complete cumulative Step 8 acceptance follow. | +| PR95 | **Recorded plan amendment, 2026-10-19 (maintainer-approved; new scope from the [PR #95 independent review](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18), not a split of rows 1–19) — scrubber residuals, before 9a:** the `api-key:Bearer`/`Basic` label erasure that leaks the token; raw `?`/`#` URL userinfo masked unless the text before it is host-shaped **and** an `=` assignment separates it from the authority `@`, applied to the LLM leaf and core's twin predicate; an extended finite scheme list (postgres, postgresql, mysql, mariadb, redis, rediss, mongodb, mongodb+srv, amqp, amqps, ssh, sftp, ldap, ldaps, smtp, smtps) masking colon-bearing userinfo for the new schemes; module-load checks of the split-credential scanner's two invariants, a start prefilter, joined-view and glued-scheme witnesses, the load-sensitive 8 MiB timing case, and the causal-control pins. One increment with ordered commits, two NEW clean complete cumulative rounds and a separate security sitting. | | 9 | **Release map/custody substrate:** #46/ADR-0111, independently reviewed after the finite graph. Upstream TS maps, sole prepack final build, pre-hash trailer removal/staging, exact tarball/CLI+engine symbolication, encrypt-once candidate with permissioned prebuild discovery/reservation → pack/reuse → smoke → archive → npm → finalization. Required artifact/platform aggregation, all rerun modes, lost-staging refusal and permission-enforcing offline API controls. Dedicated archive/crypto security sitting. Fixture proof never claims production recipient/custody configuration. | | 9a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage #46/ADR-0111 unchanged, no new contract) — release maps, final build and symbolication:** fresh uncached upstream original-TypeScript maps, the sole prepack final CLI build, final trailer removal before hashes, map and private-graph staging outside public artifacts and caches, exact tarball-to-final-JS binding, and real packed CLI and engine frame symbolication with wrong JS/map/source and missing-map negative controls. Ordinary builds remain map-free. | +| 9a.1 | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — candidate build and finalizer:** an owned isolated candidate copy, fresh uncached upstream maps, candidate-mode tsup selection with absent `sourcesContent`, the prepack finalizer (upstream and final map validation, repository-relative sources, exact trailer removal before hashes), owner-only private staging outside public artifacts and caches, closure/seal adaptation, the ordinary map-free control, the PR #95 archive/worker fold-ins, and a fail-closed `release.yml` publish guard that refuses any non-release candidate until 9d and Wave 4b custody exist. The receipt stays `compiled_package_test`. | +| 9a.2 | **Recorded split — manifest, symbolication and package-test receipt:** the private map manifest keyed by `dist/.js`, actual packed CLI and engine frame symbolication against the matching source revision, wrong JS/map/source and missing/malformed-upstream negative controls (product copies via `check:map-controls`, also a required uncached CI job), and the atomic `package_test` receipt flip across seal, binding, consumers, aggregate and both workflows. | | 9b | **Recorded split — recipient custody:** synthetic-only isolated GnuPG controls (public-only home/keyring, pinned primary and encryption-subkey fingerprints, machine-readable refusal of missing, expired, revoked or unusable recipients), encrypt once and reuse the original ciphertext. Explicit crypto/custody security sitting. Fixture proof never claims production recipient or custody configuration. | | 9c | **Recorded split — offline release state controller:** permission-enforcing offline GitHub/npm controller, permissioned prebuild discovery and reservation, and every rerun, partial, conflict and expiry state, with no overwrite, rebuild or re-encryption on uncertain history. | | 9d | **Recorded split — workflow, runbook and whole-Step-9 landing:** least-privilege actual workflow dependency graph, required artifact/platform aggregation and exact-artifact consumers, lost-staging refusal and the release runbook update. The whole-Step-9 cumulative review retains all 9a–9d changes with preservation traces into Step 8, and the dedicated archive/crypto security sitting closes the step. | | 10 | **CLI behaviour/discovery integration:** #45/#264/#265/#268/#13/#17/#18/#310/#309/#19 under ADR-0110, consuming Steps 6–9. Production owner category matrix, real root/subtree JSON help, pricing/chat aliases, non-echoing Commander refusal and signal-safe deletion consent. G15 is refuted and receives real engine/store/log preservation evidence, with no invented token coalescing. Repeat affected packed gates. | +| 10a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — diagnostic policy and output owners:** #45/#264/#265/#268 policy wiring, the production output-owner category registry with a drift test, verbose detail whose frames come from a same-build allowlist (no second build or post-finalization write), removal of the raw stack branch with G36/G46 re-scoped to its replacement, and the PR #95 lazy-dispatch cause-wrap and purity-root fold-ins. | +| 10b | **Recorded split — descriptor grammar and JSON help:** canonical chat paths and pricing spellings with working aliases and conflict refusal, non-echoing Commander refusal with the #311 static hint family (moved here from row 17; certified in 17a), excess-argument refusal, subcommand global help, real root/subtree JSON help (#13, including `help [path] --json`) and the purity guard. | +| 10c | **Recorded split — consent, G15 evidence and landing:** signal-safe key-deletion consent (#19), G15 engine/store/log preservation evidence, promotion of the implemented Steps 6–10 commands.md sections (moved here from Step 19, per landing checklist item 3) and the repeated packed gates. | | 11 | **Chat normalization and abort:** #236/#237/#242/#307 → #218 → #21/#23. Canonical token/money source with approved additive LLM export, preserved two-decimal cap formatting and real immediate abort while a turn is blocked. Audit Home's imports. | +| 11a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged, G54 moved here from row 17) — canonical token/money source:** #236/#237/#242/#307, including USD presentation of the conservative-hold disclosures, G54's character-based attachment estimate on the same helper, then #218 reseat vocabulary. | +| 11b | **Recorded split — immediate abort:** #21/#23 certification and blocked-turn abort, including Ctrl-C mapped to `/cancel` while a tool approval is pending (an approved ADR-0057 amendment: the approval is rejected, never accepted), with an approval-path security sitting. | | 12 | **Chat decomposition:** #220/#244 after preceding chat fixes. Split lifecycle/drivers, line handler, printers and reseat/clear ownership. Preserve compatibility entry points, W7/W8 receipts, unknown-usage, full-screen bounds, disclosure activation and cancellation. | +| 12a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — chat leaves:** driver contracts, pure surface decisions, printers, headless drivers, the lifecycle seam and live notices move out of `commands/chat.ts` with no behaviour change. | +| 12b | **Recorded split — line handler and wiring:** #244 line-handler decomposition, session wiring, clear/reseat ownership, the REPL loop and command bodies. `commands/chat.ts` keeps only the command entry points and their argument/dependency types, importers are repointed, and `chat.ts`-internal duplication is consolidated (Home's chat wiring moves onto the shared helpers in 13c). | | 13 | **Chat domain policy:** G30/ADR-0112 after decomposition, all fresh/resume/Home/agent/rebuild paths. Invocation-live policy owner shared by old/pending hosts, post-approval/post-prepare target guard, pinned global selection, suspension/recovery and latest-only build publication. Actual failed/overlapping Home rebuild and approval-race controls, alongside no-authority/SSRF/ports; separate egress/config-trust security sitting. | +| 13a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — pure substrate:** the DNS-hostname grammar and comparator (also rejecting `.internal`, which the SSRF floor always blocks), the strict `[chat_http].allowed_domains` key, the frozen dispatch target and the additive pre-dispatch check. | +| 13b | **Recorded split — invocation authority:** the pinned global selection and the invocation-live, monotonically narrowing HTTP authority owner wired into standalone chat fresh/resume/clear/reseat and `agent run`. | +| 13c | **Recorded split — Home integration:** Home's in-chat wiring first moves onto the shared session helpers from 12b, then shares the authority across old/pending/replacement hosts with latest-only publication; `http_request` is omitted from each turn's advertised tools while the live authority is empty or suspended (dispatch stays authoritative). Separate egress/config-trust security sitting. | | 14 | **Home controller:** certify G37, fix #281, record/test #125/D26 before #215/#243 decomposition; then #217/#222 scoped naming. Overlay discriminated union, subcontrollers, stale async work/focus/approval invariants and both live cost surfaces. | +| 14a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — correctness first:** certify G37, make the palette rejection observable (#281), settle and test #125/D26 on both live chat surfaces, and remove the unused `HomeDeps.exit` (PR #95 fold-in). | +| 14b | **Recorded split — decomposition and naming:** the overlay discriminated union with subcontroller decomposition (#215/#243) in one change, preserving the Step 8 and Step 13 invariants; the shared `[c]` reason-capture reducer on both Home and chat-ink (a call-site swap only); then #217/#222 scoped naming. | | 15 | **Viewport/input:** #58/#62/G40/G51 → #65 → #64/#66, then #63/#69/G4/G55. Grapheme-correct editing/truncation, tabs, bounded caches, safe resizing, TERM=dumb and both small-terminal surfaces, input history and mouse bounds. CJK measurement and real emitted rows. | +| 15a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — grapheme and tabs:** the shared grapheme primitive (#58/#62/G40/G51) and 8-cell tab expansion before width measurement (#65). | +| 15b | **Recorded split — width cache and resize:** the bounded 1,024-entry width memo and latest-size frame coalescing with immediate narrowing safety (#64/#66). | +| 15c | **Recorded split — terminal bounds:** 80×24 on both chat surfaces with no hidden affirmative input there or on the Home strip (#63; an approved ADR-0057 amendment), the `TERM=dumb` plain fallback including the bare-Home gate (#69; an approved ADR-0054 amendment), the 500-entry history and SGR bounds (G4/G55) and the PR #95 input-listener fold-in. Approval-path security sitting. | | 16 | **Onboarding and overlays:** #49/#52/#51/#53/#55/G5/G52/G41/G53 and #34 under ADR-0112. Real next-model/ephemeral invocation binding across clear/reseat, recoverable persistence, explicit acknowledgement/current keyless nudge, shared pre-read writer symlink refusal, surfaced typed recovery and deterministic existing-file comment-loss notices. Hidden-approval safety/color-off; keychain/config security sitting. | +| 16a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — shared writer:** the shared preference writer with pre-read link refusal and typed recovery, the acknowledgement key (#34) and path-free configuration diagnostics (extending the safe display-path discipline to `ConfigError` and `/doctor`). | +| 16b | **Recorded split — onboarding truth:** typed key availability, truthful onboarding outcomes, the current-state keyless nudge and color-off across all four clack prompt sites (#49/#52/#51/#53/#55). | +| 16c | **Recorded split — selection and overlays:** the invocation selection binding (an explicit ephemeral choice wins for the invocation; a reseat stays session-scoped), surfaced Home refusal, the one-pending-approval invariant (G5) and the overlay/color-off controls (G52/G41/G53). Whole-Step-16 keychain/config security sitting. | | 17 | **Remaining CLI/hygiene:** #24/#26/G43/G2/#12/#54/G14/#11/#47/#48/#240/#241/#311/G54/#303/#304; certify #20/G36/G46. One-shot coverage-aware cost/banner, actual-host doctor, typed faults, no silent missing-dist skip, guard misuse, attachment estimate and compiler checks. #239 remains the approved documented compatibility surface. | +| 17a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged, G54 moved to 11a) — CLI tails:** one-shot stdin hint and cost line (#24/#26), actual-host doctor (G43), typed faults (G2/#12/#54), #311 certification, consumer-audited dead exports (#240/#241), comments (G14/#11) and the four PR #95 exit-code literal sites. | +| 17b | **Recorded split — test and control hygiene:** #47/#48, catalog/compiler checks (#303/#304), certification of #20/G36/G46 and the PR #95 test-evidence and join-table nits. | | 18 | **openLocalDb LAST:** #278/#280 across every moved caller. Centralize the already-integrated classification/wrapping with fixed safe errors, attempt-all secondary cleanup, preserved causes, privacy/checkpoint notices and exact writer lifetimes. Actual command-family failure probes; no duplicate wrapper left behind. | | 19 | **Whole-wave acceptance and publication:** every item disposition; A/E/G and M2.5.5-3 evidence; ordinary/enforced coverage and required CI/build/fences/smokes/causal/platform checks at attributed source; measure and promote core's ≥90% line/branch floor to required CI as testing.md requires after Wave 3 coverage work, fixing meaningful missing coverage before closure. Two new whole-wave cumulative reviews, canonical doc/ADR landings and PR. I/whole phases/later product waves stay open. | +| 19a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — coverage:** measure core at the accepted Step 18 head, close meaningful gaps, then promote core's ≥90% line and branch floor into the required coverage job, renamed `engine coverage floor` (the maintainer updates branch protection). | +| 19b | **Recorded split — whole-wave acceptance:** every disposition, canonical landings and the Proposed-section inventory, final exact-head gates, two NEW whole-wave cumulative reviews and the PR. | Steps are dependency-safe increments, not permission to parallel-edit shared hot files. Within each row, arrows remain strict. The process/packaging sequence replaces the old fatal-before-lazy assumption: pure contracts → guarded lazy graph → native fatal integration → map/archive custody → remaining CLI behaviour. It is nineteen increments, with no parallel edits of shared hot files. Step 1 completes before all other @@ -3719,6 +3767,10 @@ preflight test pass or a clean proposal review. Approval is requested because th user explicitly requires it for new ADRs and implement-task step 3 requires it for public API/Accepted-decision/secrets changes. No product step is complete yet. +> *Superseded status, noted 2026-10-19:* this gate was satisfied on 2026-10-16 +> ([approval checkpoint](#wave-3-maintainer-approval-and-implementation-checkpoint--2026-10-16)), and +> Steps 1–8 merged as PR #95. The sentence above records the proposal stage. + #### Proposal review disposition — 2026-10-15 Two new independent complete cumulative round-1 reviews of all seven proposal paths @@ -9319,4 +9371,47 @@ sentences were made after the last review of those files; they change no behavio Author verification of the corrected tree: full local `pnpm run ci` passes end to end at `7913b7b7` (workspace lint, typecheck, tests and build; format; fences; bundle closure; smokes; the acceptance tools; and the wire, usage, filesystem and secrets mutation families); the affected suites pass with and without `CI=true GITHUB_ACTIONS=true`. The `a3e164b5` subject line (`refactor(tests)`, a non-package scope on a change that spans llm, core, cli and tools) is already published and is not rewritten. The follow-up commits are -`c64f3c39`, `f597977a`, `ebbec1c0`, `fcfe79b1` and `7913b7b7`. +`c64f3c39`, `f597977a`, `ebbec1c0`, `fcfe79b1` and `7913b7b7`. They are on `development`, pushed to +`origin/development` with the recording commit `9902e0a0`, and are **not on `main`**; they reach `main` with the next +publication checkpoint. [Push CI 39275223233](https://github.com/HodeTech/Relavium/actions/runs/39275223233) at +`9902e0a0` completed `success` on 2026-10-19; source CI certifies no later documentation commit. + +### Wave 3 Steps 9–19 preflight review and maintainer rulings — 2026-10-19 + +**Source:** `development` = `origin/development` = `9902e0a0`. Steps 9–19 had not started. Before implementation, +sixteen scoped auditors read the canonical documents for Steps 9–19 and the open PR #95 findings in full and checked +every requirement against the current source; fifteen independent adversarial verifiers re-checked each audit, and a +completeness critic compared all of them against the execution table and the current.md queue. Every finding ID in +rows 9–19 has an owner. The detailed reports stay local and Git-ignored. **No new ADR is required.** Three approved +amendments to Accepted ADRs will be appended when their increments land: ADR-0057 for Ctrl-C through the approval +swallow (11b) and for no hidden affirmative input below 80×24 (15c), and ADR-0054 for the `TERM=dumb` bare-invocation +gate (15c). Dated implementation notes follow the canonical landing checklist: ADR-0051's no-map clause (9a/9d), +ADR-0111 (9d) and ADR-0110's frame mechanism and static hint family (10a/10b). + +The maintainer approved every recommendation below on 2026-10-19. The increment table above records the resulting +plan amendment (row PR95) and the recorded splits; ID coverage is unchanged apart from PR95's new scope. + +| Area | Ruling | +| --- | --- | +| Acceptance | Each increment keeps the existing rule: implement, author gates, commit, then NEW independent complete cumulative review rounds until two consecutive rounds are clean after the last material fix. Comment- or documentation-only nits do not reset the count. Security-touching increments add a separate scoped sitting, and whole-step sittings stay where their rows require them. | +| Publication | `development` is pushed after each accepted increment, so its actual CI (including the four same-tarball platform consumers) is observed per increment. The Wave 3 PR follows Step 19. | +| Plan shape | PR95 + 9a.1, 9a.2, 9b, 9c, 9d + 10a–c + 11a–b + 12a–b + 13a–c + 14a–b + 15a–c + 16a–c + 17a–b + 18 + 19a–b (29 increments). Each split is recorded before its step starts. | +| Reassignments | G54 moves from row 17 to 11a; #311's implementation moves to 10b and its certification stays in 17a; G36/G46 are re-scoped in 10a to the branch that replaces the raw-stack branch and are certified in 17b; promotion of the Steps 6–10 commands.md sections moves from Step 19 to 10c. | +| Interim release guard | 9a.1 adds a fail-closed check to the `release.yml` publish job that refuses any non-release candidate. Consequence, acknowledged: no npm publication, including a v0.1.x patch, until 9d and Wave 4b production custody exist. | +| PR95 scrubber residuals | Fixed before 9a: the `api-key:Bearer`/`Basic` leak; raw `?`/`#` userinfo masked unless the text before it is host-shaped and an `=` assignment separates it from the authority `@`, in both the LLM leaf and core's twin predicate (core's other documented differences are recorded, not removed); an extended finite scheme list that masks colon-bearing userinfo for the new schemes; the scanner invariants, prefilter and witnesses; the load-sensitive 8 MiB timing case. LOW-1 and LOW-2 become approved deferrals with their recorded triggers. | +| Step 9 mechanics | The encrypted plaintext container is an in-house strict USTAR (regular files, repository-relative paths) recoverable with standard `gpg -d \| tar -x`; GnuPG ≥ 2.2.8 becomes a local `pnpm run ci` prerequisite and its controls fail rather than skip when it is absent; release-mode staging artifacts are retained 30 days, package-test artifacts 7; product-copy map negative controls run as `check:map-controls` locally and in a required uncached CI job; the private manifest keys final JS as `dist/.js`. | +| Step 10 compatibility | Verbose frames use a same-build allowlist (a tsup `renderChunk` substitution of a fixed-length placeholder, so no second build or post-finalization write) and appear only on classified non-infrastructure diagnostics; `--quiet` treats live progress as decoration on TTY surfaces; excess positional arguments are refused on every non-variadic command; plain `note:` stderr lines become `{type:'diagnostic', code, message}` records in JSON mode; `help [path] --json` returns the discovery DTO; human `--help` keeps Commander behaviour for unknown grammar; deprecated aliases emit one structured JSON notice or one fixed human line kept under quiet and are hidden from human help listings. | +| Steps 11–12 | The raw-µ¢ conservative-hold disclosures move to USD with their wording kept (a recorded #307 extension). Ctrl-C while a tool approval is pending maps to `/cancel`, rejecting the approval. `commands/chat.ts` keeps only its command entry points and their argument/dependency types after Step 12; `chat.ts`-internal duplication is consolidated in Step 12 and Home's chat wiring moves onto the shared helpers in 13c. | +| Step 13 | `http_request` is omitted per turn while the live authority is empty or suspended; `[chat_http].allowed_domains` also rejects `.internal`. | +| Step 14 | The `[c]` reason-capture reducer is shared by Home and chat-ink (a chat-ink call-site swap only, consistent with D18). | +| Step 15 | `TERM=dumb` (exact match) selects the plain renderers and makes the bare invocation print help; below 80×24 only Ctrl-C, Esc, `n` (reject), Ctrl-Z and mouse-report consumption act on either chat surface and on the Home strip. | +| Step 16 | An explicit ephemeral model choice wins over project, workspace and global defaults for the invocation and the UI says when it shadows a project default; a successful `/models` reseat does not update that binding. Color-off applies to all four clack prompt sites. `ConfigError` and `/doctor` stop printing absolute configuration paths. | +| Step 17 | The one-shot cost line goes to stderr after the turn settles, also after an abort or infrastructure failure once a provider call was admitted (qualified known total). Bare-Home `/doctor` reports the `chat-read-write` host profile a new chat binds plus Step 13's live authority view. The #24 stdin hint and the new notices of Steps 13, 15 and 16 are preserved under `--quiet`. | +| Step 19 | The coverage job is renamed `engine coverage floor` when core is promoted; the maintainer updates branch protection. | +| Housekeeping | 54 merged local branches were deleted with `git branch -d` and local `main` was fast-forwarded to `6d2a7372`; the restore list is kept locally. | + +PR #95 item owners: archive `TAR_FORMAT` dead branches and the `worker.mjs` ancestor loop → 9a.1; the lazy-dispatch +cause-wrap and the purity-guard root set → 10a; `HomeDeps.exit` → 14a; the kitty `data` listener → 15c; the four +exit-code literal sites → 17a; test-evidence and join-table nits → 17b; the duplicate-heading nit (with its corrected +premise) → 19b. [deferred-tasks.md](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18) +carries the same owners. diff --git a/docs/roadmap/phases/phase-2.6-conversational-authoring.md b/docs/roadmap/phases/phase-2.6-conversational-authoring.md index 9ec2254d..340b344b 100644 --- a/docs/roadmap/phases/phase-2.6-conversational-authoring.md +++ b/docs/roadmap/phases/phase-2.6-conversational-authoring.md @@ -899,6 +899,11 @@ sensitive-read floors). approval prompt even renders, defeating the ADR-0057 approval-floor investment. Wire an `allowed_domains` config path alongside the credential-resolver activation above — the same fix pattern. *(S · `apps/cli/src/chat/session-host.ts` (`chatToolPolicy`); G30)* + *(pulled forward, 2026-10-19)*: G30 is not 2.6.M work any more. The approved Wave 3 plan lands it in + [Phase 2.5.5 execution row 13](phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) under + Accepted [ADR-0112](../../decisions/0112-chat-network-grants-and-onboarding-state-are-operator-owned.md): a + global-only `[chat_http].allowed_domains` with live narrowing authority, explicitly **not** coupled to the + credential-resolver or `web_search` activation above. The prescription in this bullet is historical. - **`extra_roots`** *(pulled forward, 2026-10-12)*: invocation grants and committed narrowing selectors are accepted in [Wave 2 Step 2](phase-2.5.5-hardening-and-remediation.md#step-2-acceptance--2026-10-12). Config never grants authority. - **Tool-call rendering v2** (with 2.6.F's renderer): keep the collapsed one-line annotation as the default, add a **details view** (a `/details` toggle or transcript overlay) revealing *sanitized, From dda3b45e6bc357b1011f8f98b9de65d3fbdf37d7 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 19 Oct 2026 23:44:14 +0300 Subject: [PATCH 09/97] fix(llm): close the PR #95 scrubber residuals The PR #95 independent review left two security residuals in the diagnostic secret backstop and several scanner gaps. The maintainer ruled the policy on 2026-10-19 (Wave 3 PR95 increment); this implements it in llm, in core's journal projection twin, in the secrets causal controls (repo tooling) and in the canonical docs. - An api-key, x-api-key or x-goog-api-key value that begins with a Bearer/Basic label (also LF/TAB-split) is masked together with its token, so the header rule no longer erases the later label rules' anchor and leaks the token. Like those rules, a label that ends a line takes the next token with it, failing closed. - Raw ?/# userinfo is public only after a host-shaped prefix that an = assignment follows, in scrub-secrets and in core's bounding.ts twin: https://us?er:TOKEN@h, http://#TOKEN@h and https://my.user:1234?TOKEN@h are masked, and a host fragment without an assignment fails closed. Core keeps its documented ']' and short vanity-handle differences. - The finite URL scheme list gains sixteen connection-string schemes (postgres ... smtps). They mask colon-bearing userinfo only, so ssh://git@host stays visible; an unlisted suffix scheme still matches. - The split-credential scanner compiles its probes once. A package- internal splitProbe checks its two invariants at module load with a typed error, and a :// window skips URL starts that cannot match: an 8 MiB letter-heavy diagnostic no longer costs a RegExp attempt per character. - New witnesses, a parity test between the two maskers and nine causal controls pin each mechanism (66 controls, 416 named witnesses). The eight-MiB cases now have a 15 s case deadline, inside the causal CAUSAL_TIMEOUT_BUDGET, instead of a 5 s wall-clock assertion. - The CLI supported-leaf guard admits the leaf's own dependency-free probe module; the leaf still reaches nothing else but terminal-safe. - error-handling.md gains the canonical diagnostic secret backstop policy. built-in-tools.md and the effect journal contract record the shared ?/# predicate and its v5 consequence: a resumed effect whose args carried such userinfo now refuses instead of replaying. Refs: ADR-0113 Co-Authored-By: Claude Opus 5.5 --- .../process/supported-leaf-exports.test.ts | 7 +- docs/reference/shared-core/built-in-tools.md | 7 + docs/reference/shared-core/effect-journal.md | 8 + docs/standards/error-handling.md | 35 +++ packages/core/src/tools/bounding.test.ts | 62 +++- packages/core/src/tools/bounding.ts | 26 +- packages/llm/src/scrub-secrets.test.ts | 273 ++++++++++++++++-- packages/llm/src/scrub-secrets.ts | 121 +++++--- .../llm/src/split-credential-probe.test.ts | 70 +++++ packages/llm/src/split-credential-probe.ts | 78 +++++ tools/causal-controls/provenance.test.mjs | 17 +- tools/secrets-controls/README.md | 25 +- tools/secrets-controls/mutations.mjs | 140 ++++++++- 13 files changed, 783 insertions(+), 86 deletions(-) create mode 100644 packages/llm/src/split-credential-probe.test.ts create mode 100644 packages/llm/src/split-credential-probe.ts diff --git a/apps/cli/src/process/supported-leaf-exports.test.ts b/apps/cli/src/process/supported-leaf-exports.test.ts index 57da3b71..d50e5336 100644 --- a/apps/cli/src/process/supported-leaf-exports.test.ts +++ b/apps/cli/src/process/supported-leaf-exports.test.ts @@ -70,10 +70,13 @@ describe('supported compiled dependency leaves', () => { } }); - it('loads the actual supported compiled leaf with only terminal-safe reachable', () => { + it('loads the actual supported compiled leaf with only terminal-safe and its own probe reachable', () => { const require = createRequire(import.meta.url); + const leaf = pathToFileURL(require.resolve('@relavium/llm/scrub-secrets')).href; const allowed = [ - pathToFileURL(require.resolve('@relavium/llm/scrub-secrets')).href, + leaf, + // The leaf's dependency-free, package-internal split-credential probe (no package export). + new URL('./split-credential-probe.js', leaf).href, pathToFileURL(require.resolve('@relavium/shared/terminal-safe')).href, ]; // An isolated ESM loader refuses every other module, including schema/provider barrels. diff --git a/docs/reference/shared-core/built-in-tools.md b/docs/reference/shared-core/built-in-tools.md index 09662fbb..7a7405fa 100644 --- a/docs/reference/shared-core/built-in-tools.md +++ b/docs/reference/shared-core/built-in-tools.md @@ -93,6 +93,13 @@ This is conservative pattern masking, not proof that arbitrary sensitive content absent; raw dispatch/model results, full spills and historical rows retain their existing content-sensitive handling. +A raw `?` or `#` before a URL authority's `@` begins public query or fragment text only when +the text before it is host-shaped (a dotted name or IPv4, `localhost` or a bracketed IP literal, +with an optional port) **and** an `=` assignment follows it; otherwise the userinfo is masked. +This is the same predicate as the [LLM diagnostic backstop](../../standards/error-handling.md#diagnostic-secret-backstop). +Two projection-only differences remain by design: `]` ends an authority, and a userinfo +with no password that is shorter than 16 characters is kept as a vanity handle. + LF/TAB inside a recognized token-shaped credential extends the masked original span. An ambiguous token-alphabet continuation after LF/TAB is conservatively masked with that credential. Ordinary spaces and punctuation separate public prose; the masker diff --git a/docs/reference/shared-core/effect-journal.md b/docs/reference/shared-core/effect-journal.md index f1c3a8db..3d41da84 100644 --- a/docs/reference/shared-core/effect-journal.md +++ b/docs/reference/shared-core/effect-journal.md @@ -376,6 +376,14 @@ through the actual closing quote. A recognized value must not leave a credential in the projection. This remains a finite shape detector, not a claim to identify arbitrary unlabelled secrets. Existing old digests are not retroactively erased or rewritten. +A raw `?` or `#` before the authority's `@` is public only after a host-shaped prefix that an +`=` assignment follows ([built-in tools](built-in-tools.md#finite-diagnostic-credential-masking)). +Wave 3's PR95 increment tightened that predicate without a new digest domain. It changes a +`v5:` projection only where such userinfo was previously treated as public: a resumed run whose +committed effect carried one now computes a different digest and refuses matching prepare +instead of replaying, which is the fail-closed outcome of any digest difference. Every other +`v5:` row keeps its digest. + A recognized structural credential key masks its entire value before traversal. A partial inner text match does not make the remaining value public. Only an anchored Bearer, Basic or Token scheme survives as its canonical label followed by `[redacted]`; no original diff --git a/docs/standards/error-handling.md b/docs/standards/error-handling.md index abaf37f5..c7c38b4c 100644 --- a/docs/standards/error-handling.md +++ b/docs/standards/error-handling.md @@ -120,6 +120,41 @@ owns the fixture-pinned classification and custom-endpoint downgrade. Actual usa with a failed native response is still priced and recorded; a reserved estimate is never reported as actual usage. Surfaces supply their own policy-aware commands. +### Diagnostic secret backstop + +`makeLlmError` passes every `message` and `code` through `scrubSecrets` +(`packages/llm/src/scrub-secrets.ts`), and the CLI's display sanitizer composes the same function. +It is a defense-in-depth **backstop**: the adapter still owns passing an already-redacted +message, and the backstop masks a finite set of secret **shapes**, not arbitrary sensitive +content. + +- **Shapes.** OpenAI/Anthropic `sk-…` and Google `AIza…` key prefixes; a finite list of literal + secret query names; the exact `api-key`, `x-api-key` and `x-goog-api-key` header names (a + value that begins with a `Bearer`/`Basic` label is masked together with its token); and + `Bearer`/`Basic` authentication values. +- **URL userinfo.** The schemes `http(s)`, `ws(s)` and `ftp(s)` mask any userinfo. The + connection-string schemes `postgres`, `postgresql`, `mysql`, `mariadb`, `redis`, `rediss`, + `mongodb`, `mongodb+srv`, `amqp`, `amqps`, `ssh`, `sftp`, `ldap`, `ldaps`, `smtp` and `smtps` + mask colon-bearing userinfo (`user:pass@`, `:pass@`); a bare user such as `ssh://git@host` + stays visible. A raw `?` or `#` before the authority `@` begins public query or fragment + text only when the text before it is host-shaped (a dotted name or IPv4, `localhost` or a + bracketed IP literal, with an optional port) **and** an `=` assignment follows it, as in + `https://host.example?contact=a@b.example`; otherwise the userinfo is masked. Core's journal + projection applies the same `?`/`#` predicate + ([built-in tools](../reference/shared-core/built-in-tools.md#finite-diagnostic-credential-masking)). +- **Split and normalized forms.** Each rule also probes the terminal-normalized text and the + LF/TAB-joined view at original-text boundaries; a credential revealed only by + normalization or by an LF/TAB split masks the **whole** diagnostic. +- **Documented trade-offs, all failing closed.** A scheme glued to a word character is masked in + place; an unlisted scheme ending in a listed one (`gftp://`) matches by its suffix; prose + that the whitespace-stripped residual probe glues to a path-less URL and a later `@` blanks the + whole diagnostic; a host fragment with no assignment (`https://host.example#sec@x`) and a + single-label host query are masked; and a `Bearer`/`Basic` label that ends a line takes the + next token with it. +- **Limits.** Malformed URLs (`https:/u:p@h`), a raw `/` inside a password, invisible characters + inside a scheme and opaque fragments separated by ordinary spaces are outside the finite + shapes. + ## No silent catches - A `catch` block must do one of: handle the error meaningfully, enrich and rethrow it, or diff --git a/packages/core/src/tools/bounding.test.ts b/packages/core/src/tools/bounding.test.ts index 05c28d8e..19a34b4b 100644 --- a/packages/core/src/tools/bounding.test.ts +++ b/packages/core/src/tools/bounding.test.ts @@ -1,3 +1,4 @@ +import { scrubSecrets } from '@relavium/llm/scrub-secrets'; import type { AbortSignalLike } from '@relavium/shared'; import { describe, expect, it, vi } from 'vitest'; @@ -629,9 +630,13 @@ describe('redactSecretShapedText — URL userinfo (ADR-0080 §11)', () => { for (const text of [ 'https://vanity@api.invalid/x', 'https://api.invalid?public=alice@example.invalid', - 'https://api.invalid#alice@example.invalid', + 'https://api.invalid#contact=alice@example.invalid', ]) expect(redactSecretShapedText(text)).toBe(text); + // A host fragment with no assignment may be a malformed password: masked, failing closed. + expect(redactSecretShapedText('https://api.invalid#alice@example.invalid')).toBe( + 'https://[redacted]@example.invalid', + ); expect(redactSecretShapedText(`https://${'u'.repeat(600)}@api.invalid/x`)).toBe( 'https://[redacted]@api.invalid/x', ); @@ -1154,3 +1159,58 @@ describe('PR95 B2 projection boundaries and admitted-scale controls', () => { expectCredentialProjection(redactSecretShapedText(publicText)).toBe(publicText); }); }); + +describe('PR95 raw ?/# userinfo in the journal projection', () => { + const userinfoContact = (value: unknown) => expect(value, 'SECRETS_B2_CORE_USERINFO_CONTACT'); + + it.each([ + ['https://us?er:TOKEN@h.example/', 'https://[redacted]@h.example/'], + ['https://us?er=1:TOKEN@h.example/', 'https://[redacted]@h.example/'], + ])('masks userinfo whose raw ?/# follows no host-shaped prefix: %s', (input, expected) => { + userinfoContact(redactSecretShapedText(input)).toBe(expected); + }); + + it.each([ + ['https://my.user:1234?TOKEN@h.example/', 'https://[redacted]@h.example/'], + ['https://provider.example#section@x.example', 'https://[redacted]@x.example'], + ])('masks a host-shaped prefix that no assignment follows: %s', (input, expected) => { + userinfoContact(redactSecretShapedText(input)).toBe(expected); + }); + + it.each(['?', '#'])('retains public %s contact text after a host-shaped prefix', (separator) => { + for (const host of [ + 'provider.invalid', + 'provider.invalid:8443', + 'provider.invalid.', + '10.0.0.1', + 'localhost:3000', + ]) { + const input = `connect https://${host}${separator}contact=alice@example.invalid failed`; + expect(redactSecretShapedText(input)).toBe(input); + } + }); + + it('keeps its documented short colonless-handle difference from the LLM backstop', () => { + // Core retains a colonless userinfo under 16 characters; the LLM diagnostic backstop masks it. + expect(redactSecretShapedText('http://#TOKEN@h.example')).toBe('http://#TOKEN@h.example'); + expect(scrubSecrets('http://#TOKEN@h.example')).toBe('http://[REDACTED]@h.example'); + }); + + it('agrees with the LLM backstop on every raw ?/# decision outside those differences', () => { + for (const input of [ + 'https://alice:pa?ss@provider.invalid/v1', + 'https://alice:pa#ss@provider.invalid/v1', + 'https://alice:pa?ss=value@provider.invalid/v1', + 'https://us?er:TOKEN@h.example/', + 'https://us?er=1:TOKEN@h.example/', + 'https://my.user:1234?TOKEN@h.example/', + 'https://provider.example#section@x.example', + 'https://provider.invalid?contact=alice@example.invalid', + 'https://provider.invalid:8443#contact=alice@example.invalid', + 'https://localhost:3000?contact=alice@example.invalid', + 'https://10.0.0.1#contact=alice@example.invalid', + ]) { + expect(redactSecretShapedText(input) === input, input).toBe(scrubSecrets(input) === input); + } + }); +}); diff --git a/packages/core/src/tools/bounding.ts b/packages/core/src/tools/bounding.ts index aa82048d..428f7993 100644 --- a/packages/core/src/tools/bounding.ts +++ b/packages/core/src/tools/bounding.ts @@ -431,18 +431,23 @@ function redactOpaqueQueryValues(text: string): string { return parts.join(''); } +/** A host-shaped prefix: a dotted name (or IPv4), `localhost` or a bracketed IP literal, with an optional port. */ +const HOST_SHAPED_AUTHORITY = + /^(?:[a-z\d-]+(?:\.[a-z\d-]+)+\.?|localhost|\[[\da-f:.]+\])(?::\d{1,5})?$/i; + /** * One whole authority, with its last `@` separating userinfo from the host. Prefix-only * matches can mask the first password fragment and leave another `@` plus a credential * tail in the durable digest. Paths and JSON delimiters stop the scan; queries/fragments - * after an existing userinfo separator remain public. Before that separator, raw ?/# in a - * colon-bearing password are masked, except an explicit public host:port prefix; a short colonless vanity handle retains the existing non-secret policy. + * after an existing userinfo separator remain public. Before that separator, a raw ?/# + * begins public query/fragment text only when the text before it is host-shaped **and** an + * `=` assignment follows it; anything else may be a malformed username or password. This is + * the same predicate as the LLM diagnostic backstop (`packages/llm/src/scrub-secrets.ts`). + * Two core-only differences remain by design: `]` ends an authority, and a userinfo with no + * password that is shorter than 16 characters is a vanity handle under the existing non-secret policy. */ -function isPublicHostPort(text: string): boolean { - const colon = text.lastIndexOf(':'); - if (colon < 0 || !/^\d{1,5}$/.test(text.slice(colon + 1))) return false; - const hostname = text.slice(0, colon); - return hostname.includes('.') || hostname === 'localhost' || /^\[[\da-f:]+\]$/i.test(hostname); +function isPublicQueryContact(userinfo: string, query: number): boolean { + return HOST_SHAPED_AUTHORITY.test(userinfo.slice(0, query)) && userinfo.includes('=', query + 1); } function visitUrlUserinfo(text: string, mask: SecretSpanVisitor): void { @@ -461,10 +466,9 @@ function visitUrlUserinfo(text: string, mask: SecretSpanVisitor): void { const userinfo = authority.slice(0, at); const colon = userinfo.indexOf(':'); const query = userinfo.search(/[?#]/); - // Raw ?/# can belong to a malformed password. An explicit host:port prefix instead - // introduces a public query/fragment; its contact values must not become userinfo. - if (query >= 0 && (colon < 0 || colon > query || isPublicHostPort(userinfo.slice(0, query)))) - continue; + // Raw ?/# can belong to a malformed username or password. A host-shaped prefix followed by + // an assignment instead introduces a public query/fragment whose contact must not become userinfo. + if (query >= 0 && isPublicQueryContact(userinfo, query)) continue; const hasPassword = colon >= 0 && colon < userinfo.length - 1; if (!hasPassword && userinfo.length < 16) continue; mask(start, start + at, '[redacted]'); diff --git a/packages/llm/src/scrub-secrets.test.ts b/packages/llm/src/scrub-secrets.test.ts index c26de808..a8044c78 100644 --- a/packages/llm/src/scrub-secrets.test.ts +++ b/packages/llm/src/scrub-secrets.test.ts @@ -8,6 +8,13 @@ import { scrubSecrets } from './scrub-secrets.js'; const key = ['sk-', 'ant-', 'api03-', 'AbCdEf0123456789xyz'].join(''); const googleKey = ['AI', 'za', 'SyA1234567890abcdefghijklmnopqrstuv'].join(''); +/** + * Eight-MiB cases take about a second. Their bound is this case deadline, far below the minutes a + * quadratic scan would need, rather than a wall-clock assertion that a loaded runner can trip. It is + * also the largest case budget the secrets causal-control phases admit (`CAUSAL_TIMEOUT_BUDGET`). + */ +const ADMITTED_SIZE_CASE_TIMEOUT_MS = 15_000; + const compatibleDiagnostics = [ { input: `invalid ${key}; retry later`, expected: 'invalid [REDACTED]; retry later' }, { input: `invalid ${googleKey}; retry later`, expected: 'invalid [REDACTED]; retry later' }, @@ -172,7 +179,7 @@ describe('finite diagnostic credential backstop regressions (PR #95 M2)', () => // Documents, and does not endorse, two consequences of a URL rule with no word boundary. Both fail // closed: they blank or widen a diagnostic rather than reveal one. The whitespace-stripped residual probe - // glues the prose before a URL to its scheme, and a longer scheme name matches by its suffix. + // glues the prose before a URL to its scheme, and an unlisted longer scheme name matches by its suffix. it.each([ { input: 'Error: ECONNREFUSED http://127.0.0.1:11434 contact admin@corp', @@ -182,7 +189,11 @@ describe('finite diagnostic credential backstop regressions (PR #95 M2)', () => input: 'MCP server at ws://127.0.0.1:8080 refused; email support@vendor.com', expected: '[REDACTED]', }, - { input: 'sftp://scm@code.example/x', expected: 'sftp://[REDACTED]@code.example/x' }, + { + input: 'Error: connect ECONNREFUSED postgres://db.example:5432 contact dba@corp', + expected: '[REDACTED]', + }, + { input: 'gftp://scm@code.example/x', expected: 'gftp://[REDACTED]@code.example/x' }, ])('fails closed on the documented trade-off $input', ({ input, expected }) => { expect(scrubSecrets(input)).toBe(expected); }); @@ -324,16 +335,38 @@ describe('PR95 B2 finite joined-view boundaries', () => { }, ); - it('bounds admitted-size failed header and URL preparation', () => { - const started = performance.now(); - for (const fragment of ['wordx-\tapi-\nkey ', 'https:\n//provider.invalid/']) { - const input = fragment.repeat(Math.floor((8 * 1024 * 1024 - 256) / fragment.length)); - const output = scrubSecrets(input); - expect(output.length, 'SECRETS_B2_HEADER_PREPARATION').toBe(input.length); - expect(output === input, 'SECRETS_B2_HEADER_PREPARATION').toBe(true); - } - expect(performance.now() - started, 'SECRETS_B2_HEADER_PREPARATION').toBeLessThan(5_000); - }); + it( + 'bounds admitted-size failed header and URL preparation', + () => { + for (const fragment of ['wordx-\tapi-\nkey ', 'https:\n//provider.invalid/']) { + const input = fragment.repeat(Math.floor((8 * 1024 * 1024 - 256) / fragment.length)); + const output = scrubSecrets(input); + expect(output.length, 'SECRETS_B2_HEADER_PREPARATION').toBe(input.length); + expect(output === input, 'SECRETS_B2_HEADER_PREPARATION').toBe(true); + } + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); + + it( + 'reaches the split URL scan over admitted-size letter-heavy text', + () => { + const letters = 8 * 1024 * 1024 - 256; + for (const filler of ['a', 'hwf']) { + const tail = filler.repeat(Math.floor(letters / filler.length)); + for (const [input, expected] of [ + [`a\nhttps://u@h ${tail}`, `a\nhttps://[REDACTED]@h ${tail}`], + [`${tail}\nhttps://u@h`, `${tail}\nhttps://[REDACTED]@h`], + ] as const) { + const output = scrubSecrets(input); + // Bounded evidence: a regression must not serialize eight MiB into its report. + expect(output.length, 'SECRETS_B2_URL_SCAN').toBe(expected.length); + expect(output === expected, 'SECRETS_B2_URL_SCAN').toBe(true); + } + } + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); it.each([ ['sk-proj-', '\n'], @@ -348,17 +381,19 @@ describe('PR95 B2 finite joined-view boundaries', () => { expect(error.code, 'SECRETS_B2_ORIGINAL_BOUNDARY').toBe('[REDACTED]'); }); - it('bounds admitted-size embedded prefixes before a valid original boundary', () => { - const fragment = `wordsk-proj-${'A'.repeat(52)}\n`; - const publicPrefix = fragment.repeat(Math.floor((8 * 1024 * 1024 - 256) / fragment.length)); - const input = `${publicPrefix}sk-proj-${'B'.repeat(32)}\t${'C'.repeat(32)}; public=A`; - const started = performance.now(); - const output = scrubSecrets(input); - // Keep removal evidence bounded: a regression must not serialize eight MiB into its report. - expect(output.length, 'SECRETS_B2_ORIGINAL_BOUNDARY').toBe('[REDACTED]'.length); - expect(output, 'SECRETS_B2_ORIGINAL_BOUNDARY').toBe('[REDACTED]'); - expect(performance.now() - started, 'SECRETS_B2_ORIGINAL_BOUNDARY').toBeLessThan(5_000); - }); + it( + 'bounds admitted-size embedded prefixes before a valid original boundary', + () => { + const fragment = `wordsk-proj-${'A'.repeat(52)}\n`; + const publicPrefix = fragment.repeat(Math.floor((8 * 1024 * 1024 - 256) / fragment.length)); + const input = `${publicPrefix}sk-proj-${'B'.repeat(32)}\t${'C'.repeat(32)}; public=A`; + const output = scrubSecrets(input); + // Keep removal evidence bounded: a regression must not serialize eight MiB into its report. + expect(output.length, 'SECRETS_B2_ORIGINAL_BOUNDARY').toBe('[REDACTED]'.length); + expect(output, 'SECRETS_B2_ORIGINAL_BOUNDARY').toBe('[REDACTED]'); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); it('does not invent a key boundary in a public word with a separate newline', () => { const input = `wordsk-proj-${'A'.repeat(32)};\nordinary`; @@ -391,3 +426,195 @@ describe('PR95 B2 finite joined-view boundaries', () => { ).toBe('[REDACTED]\n request; retry later'); }); }); + +const COLONLESS_SCHEMES = ['http', 'https', 'ws', 'wss', 'ftp', 'ftps']; +const CONNECTION_SCHEMES = [ + 'postgres', + 'postgresql', + 'mysql', + 'mariadb', + 'redis', + 'rediss', + 'mongodb', + 'mongodb+srv', + 'amqp', + 'amqps', + 'ssh', + 'sftp', + 'ldap', + 'ldaps', + 'smtp', + 'smtps', +]; + +describe('PR95 header values that begin with a Bearer/Basic label', () => { + const token = ['abcdef', 'ghijkl', '123456'].join(''); + const headerSchemeLabel = (value: unknown) => expect(value, 'SECRETS_B2_HEADER_SCHEME_LABEL'); + + it.each(['api-key', 'x-api-key', 'x-goog-api-key', 'X-Api-Key'])( + 'masks the token after a scheme label in the %s header', + (name) => { + for (const label of ['Bearer', 'Basic', 'bearer', 'BASIC']) { + for (const [input, expected] of [ + [`${name}:${label} ${token}`, `${name}:[REDACTED]`], + [`header ${name}: ${label} ${token} rejected`, `header ${name}: [REDACTED] rejected`], + [`${name}:\t${label}\t${token}`, `${name}:\t[REDACTED]`], + [ + `{"${name}":"${label} ${token}","ordinary":"kept"}`, + `{"${name}":"[REDACTED]","ordinary":"kept"}`, + ], + ] as const) { + const output = scrubSecrets(input); + headerSchemeLabel(output).toBe(expected); + headerSchemeLabel(output.includes(token)).toBe(false); + } + } + }, + ); + + it('masks every LF/TAB split position inside a scheme label after a header', () => { + for (const label of ['Bearer', 'Basic']) { + for (const separator of ['\n', '\t', '\n\t']) { + for (let position = 1; position < label.length; position += 1) { + const input = `api-key:${label.slice(0, position)}${separator}${label.slice(position)} ${token}`; + headerSchemeLabel(scrubSecrets(input)).toBe('api-key:[REDACTED]'); + } + } + } + }); + + it('masks the next token, failing closed, when a scheme label ends the line', () => { + headerSchemeLabel(scrubSecrets('api-key: Bearer\nrequest failed')).toBe( + 'api-key: [REDACTED] failed', + ); + expect(scrubSecrets('api-key: Bearer')).toBe('api-key: [REDACTED]'); + }); + + it('keeps a token masked when the scheme label precedes the header name', () => { + expect(scrubSecrets(`Bearer api-key: ${token}`)).toBe('Bearer [REDACTED] [REDACTED]'); + }); +}); + +describe('PR95 raw ?/# userinfo: public only after a host-shaped prefix and an assignment', () => { + const userinfoContact = (value: unknown) => expect(value, 'SECRETS_B2_USERINFO_CONTACT'); + + it.each([ + ['https://us?er:TOKEN@h.example/', 'https://[REDACTED]@h.example/'], + ['http://#TOKEN@h.example', 'http://[REDACTED]@h.example'], + ['https://us?er=1:TOKEN@h.example/', 'https://[REDACTED]@h.example/'], + ])('masks userinfo whose raw ?/# follows no host-shaped prefix: %s', (input, expected) => { + userinfoContact(scrubSecrets(input)).toBe(expected); + }); + + it.each([ + ['https://my.user:1234?TOKEN@h.example/', 'https://[REDACTED]@h.example/'], + ['https://host.example#sec@x.example', 'https://[REDACTED]@x.example'], + ])('masks a host-shaped prefix that no assignment follows: %s', (input, expected) => { + userinfoContact(scrubSecrets(input)).toBe(expected); + }); + + it.each(['?', '#'])('retains public %s contact text after a host-shaped prefix', (separator) => { + for (const host of [ + 'provider.invalid', + 'provider.invalid:8443', + 'provider.invalid.', + '10.0.0.1', + 'localhost:3000', + '[::1]', + '[::1]:8443', + '[::ffff:10.0.0.1]', + ]) { + const input = `connect https://${host}${separator}contact=alice@example.invalid failed`; + expect(scrubSecrets(input)).toBe(input); + } + }); + + it('masks a single-label host query, failing closed', () => { + expect(scrubSecrets('https://intranet?contact=alice@example.invalid')).toBe( + 'https://[REDACTED]@example.invalid', + ); + }); + + it.each(['\n', '\t'])('keeps the predicate across an LF/TAB split (%j)', (separator) => { + userinfoContact(scrubSecrets(`https://us?e${separator}r:TOKEN@h.example/`)).toBe('[REDACTED]'); + const contact = `https://provider.invalid?con${separator}tact=alice@example.invalid`; + expect(scrubSecrets(contact)).toBe(contact); + }); +}); + +describe('PR95 connection-string URL schemes', () => { + const connectionScheme = (value: unknown) => expect(value, 'SECRETS_B2_CONNECTION_SCHEME'); + + it.each(CONNECTION_SCHEMES)('masks colon-bearing %s userinfo and keeps a bare user', (scheme) => { + for (const spelling of [scheme, scheme.toUpperCase()]) { + connectionScheme(scrubSecrets(`connect ${spelling}://user:opaque@db.example/x failed`)).toBe( + `connect ${spelling}://[REDACTED]@db.example/x failed`, + ); + connectionScheme(scrubSecrets(`${spelling}://:opaque@db.example`)).toBe( + `${spelling}://[REDACTED]@db.example`, + ); + connectionScheme(scrubSecrets(`x${spelling}://user:opaque@db.example/x`)).toBe( + `x${spelling}://[REDACTED]@db.example/x`, + ); + const bareUser = `${spelling}://admin@db.example/x`; + expect(scrubSecrets(bareUser)).toBe(bareUser); + } + }); + + it('masks a multi-host connection string and keeps a git+ssh user', () => { + connectionScheme(scrubSecrets('mongodb://user:opaque@h1.example,h2.example/db')).toBe( + 'mongodb://[REDACTED]@h1.example,h2.example/db', + ); + expect(scrubSecrets('git+ssh://git@github.com/org/repo')).toBe( + 'git+ssh://git@github.com/org/repo', + ); + }); + + it('keeps masking colonless userinfo for the original schemes', () => { + for (const scheme of COLONLESS_SCHEMES) { + expect(scrubSecrets(`${scheme}://opaque@provider.invalid/x`)).toBe( + `${scheme}://[REDACTED]@provider.invalid/x`, + ); + } + }); +}); + +describe('PR95 joined-view and glued-scheme witnesses', () => { + // An inner pre-masked span splits the whitespace-stripped residual probe, so only the joined + // scanner can see each split credential below. + it.each([...COLONLESS_SCHEMES, ...CONNECTION_SCHEMES])( + 'masks a split %s credential around an inner masked span', + (scheme) => { + for (const separator of ['\n', '\t']) { + expect( + scrubSecrets(`${scheme}://u${separator}ser:pa[REDACTED]ss@h.example/x`), + 'SECRETS_B2_URL_JOINED', + ).toBe('[REDACTED]'); + } + }, + ); + + it.each(['?', '#'])( + 'masks a split credential holding a raw %s and an inner masked span', + (raw) => { + for (const separator of ['\n', '\t']) { + expect( + scrubSecrets(`https://u${separator}ser:pa${raw}ss[REDACTED]x@h.example/v1`), + 'SECRETS_B2_URL_JOINED', + ).toBe('[REDACTED]'); + } + }, + ); + + it.each(['\n', '\t'])( + 'masks a glued scheme whose split credential holds a raw @ (%j)', + (separator) => { + for (const scheme of ['https', 'postgres']) { + expect( + scrubSecrets(`x${scheme}://user:p@ss${separator}word@provider.invalid/v1`), + 'SECRETS_B2_URL_GLUED_SCHEME', + ).toBe('[REDACTED]'); + } + }, + ); +}); diff --git a/packages/llm/src/scrub-secrets.ts b/packages/llm/src/scrub-secrets.ts index 6c057665..c34ff3c8 100644 --- a/packages/llm/src/scrub-secrets.ts +++ b/packages/llm/src/scrub-secrets.ts @@ -1,12 +1,14 @@ import { stripTerminalControls } from '@relavium/shared/terminal-safe'; +import { splitProbe, type SplitProbe, type SplitProbeAnchor } from './split-credential-probe.js'; + /** * Defense-in-depth secret scrub applied at the one `makeLlmError` choke point. The CALLER (adapter) is * still responsible for passing an already-redacted message; this is the **structural backstop** — like * the engine's `deInlineMedia` pass, leak-freedom here is an active transform at one boundary, not * per-adapter discipline. It masks finite common secret **shapes**: prefixed keys, literal secret query - * names, URL userinfo and finite API-key/Bearer/Basic headers. Each non-query rule probes the original diagnostic - * independently, including terminal-normalized and LF/TAB-joined views, before any + * names, URL userinfo of a finite scheme list and finite API-key/Bearer/Basic headers. Each non-query rule probes + * the original diagnostic independently, including terminal-normalized and LF/TAB-joined views, before any * other rule can erase its identifying context. A match revealed by normalization masks the whole * diagnostic. The finite patterns check boundaries in the original text (the URL rule has none, like the * pre-PR matcher, so a scheme glued to a word character is masked in place) and preserve already-masked spans; @@ -15,6 +17,7 @@ import { stripTerminalControls } from '@relavium/shared/terminal-safe'; * finite residual check, with no parsing or external calls on the error path. * Asserted by `llm-error.test.ts` (the backstop) and each adapter's * secret-safety test (a planted secret → a secret-free surfaced `LlmError`); see security-review.md. + * The canonical policy is error-handling.md's diagnostic secret backstop. */ export function scrubSecrets(text: string): string { return scrubQuerySecrets(scrubNonQuerySecrets(text)); @@ -25,21 +28,35 @@ interface SecretRule { readonly replacement: string | ((match: string) => string); readonly joinedPattern?: RegExp; readonly boundary?: RegExp; + /** Split-scanner prefilter: a match must reach this literal within its window (see `splitProbe`). */ + readonly anchor?: SplitProbeAnchor; } /** * Deliberately NOT word-bounded: the pre-PR http(s) matcher masked `xhttps://user:pass@host`, and a `\b` - * here would let any scheme glued to a preceding word character through unmasked. This PR added - * ws/wss/ftp/ftps under the same rule. The rule has no `\b` prefix, so it supplies its own joined form - * (the split-credential scanner would otherwise strip two characters that are not there) and its - * `boundary` blocks no start. Consequences, pinned by tests and fail-closed: a longer scheme name - * ending in one of these (`sftp://u@h`) matches by suffix, and the whitespace-stripped residual probe - * turns a path-less URL, prose and a later `@` into a whole-diagnostic `[REDACTED]` (the pre-PR matcher did - * so for http(s); dropping `\b` extends it to prose that stripping glues to the scheme). A credential - * split by a plain SPACE that also holds a raw `@` remains the documented opaque-fragments-across-spaces - * limit, as it was for the pre-PR matcher. + * here would let any scheme glued to a preceding word character through unmasked. PR #95 added + * ws/wss/ftp/ftps under the same rule, and the PR95 increment added the connection-string schemes + * (postgres(ql), mysql, mariadb, redis(s), mongodb(+srv), amqp(s), ssh, sftp, ldap(s), smtp(s)), whose + * colonless userinfo names a user and stays visible (see `maskUrlUserinfo`). The rule has no `\b` prefix, + * so it supplies its own joined form (the split-credential scanner would otherwise strip two characters + * that are not there) and its `boundary` blocks no start. Consequences, pinned by tests and fail-closed: an + * unlisted longer scheme name ending in a listed one (`gftp://u@h`) matches by suffix, and the + * whitespace-stripped residual probe turns a path-less URL, prose and a later `@` into a whole-diagnostic + * `[REDACTED]` (the pre-PR matcher did so for http(s); dropping `\b` extends it to prose that stripping glues + * to the scheme). A credential split by a plain SPACE that also holds a raw `@` remains the documented + * opaque-fragments-across-spaces limit, as it was for the pre-PR matcher. + */ +const URL_USERINFO_PATTERN = + /(?:https?|wss?|ftps?|postgres(?:ql)?|mysql|mariadb|rediss?|mongodb(?:\+srv)?|amqps?|ssh|sftp|ldaps?|smtps?):\/\/[^/\s]*@/gi; + +/** The schemes whose colonless userinfo is masked too; every other listed scheme needs a `:` in it. */ +const COLONLESS_USERINFO_SCHEME = /^(?:https?|wss?|ftps?)$/i; + +/** + * Every listed scheme ends within this many joined characters before its `://`, so the split scanner + * tries no URL start farther away. It is generous on purpose: the longest scheme (`mongodb+srv`) has 11. */ -const URL_USERINFO_PATTERN = /(?:https?|wss?|ftps?):\/\/[^/\s]*@/gi; +const URL_SCHEME_WINDOW = 32; /** A start boundary that never matches: no preceding character makes a start ineligible. */ const NO_BOUNDARY = /(?!)/; @@ -58,16 +75,21 @@ const NON_QUERY_SECRET_RULES: readonly SecretRule[] = [ boundary: /\w/, }, // Finite authority schemes, not URL validation. Raw ?/# before the first @ may belong - // to a password; after an existing @ they begin public query/fragment text. + // to a username or password; after an existing @ they begin public query/fragment text. { pattern: URL_USERINFO_PATTERN, joinedPattern: URL_USERINFO_PATTERN, replacement: maskUrlUserinfo, boundary: NO_BOUNDARY, + anchor: { text: '://', window: URL_SCHEME_WINDOW }, }, // Exact literal header names, also in quoted diagnostic objects. No arbitrary header discovery. + // A value may begin with a (possibly LF/TAB-split) Bearer/Basic label: the label and its token are + // masked together, so the later scheme-label rules cannot lose their anchor to this one. Like those + // rules, whitespace after the label may be a line break, and the next token is masked fail-closed. { - pattern: /(^|[^A-Za-z0-9_-])((?:x-)?api-key|x-goog-api-key)(["']?\s*:\s*["']?)[^\s"',;}]+/gi, + pattern: + /(^|[^A-Za-z0-9_-])((?:x-)?api-key|x-goog-api-key)(["']?\s*:\s*["']?)(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r\s+|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c\s+)?[^\s"',;}]+/gi, replacement: '$1$2$3[REDACTED]', joinedPattern: /(?:(?:x-)?api-key|x-goog-api-key)["']?\s*:/gi, boundary: /[A-Za-z0-9_-]/, @@ -81,16 +103,38 @@ const NON_QUERY_SECRET_RULES: readonly SecretRule[] = [ { pattern: /\bB[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c\s+\S+/gi, replacement: 'Basic [REDACTED]' }, ]; +/** + * Each boundary-bearing rule compiled once for the split scanner. `splitProbe` checks the scanner's + * two invariants here, at module load, so a misconfigured rule fails fast with a typed error instead of + * a `SyntaxError` inside `makeLlmError` on some later LF/TAB diagnostic. + */ +const SPLIT_PROBES: ReadonlyMap = new Map( + NON_QUERY_SECRET_RULES.flatMap((rule) => + rule.boundary === undefined + ? [] + : [[rule, splitProbe({ ...rule, boundary: rule.boundary })] as const], + ), +); + /** Reuse the display boundary's platform-free normalization without changing benign output. */ function normalizeSecretProbe(text: string): string { return stripTerminalControls(text); } -function isPublicHostPort(text: string): boolean { - const colon = text.lastIndexOf(':'); - if (colon < 0 || !/^\d{1,5}$/.test(text.slice(colon + 1))) return false; - const hostname = text.slice(0, colon); - return hostname.includes('.') || hostname === 'localhost' || /^\[[\da-f:]+\]$/i.test(hostname); +/** A host-shaped prefix: a dotted name (or IPv4), `localhost` or a bracketed IP literal, with an optional port. */ +const HOST_SHAPED_AUTHORITY = + /^(?:[a-z\d-]+(?:\.[a-z\d-]+)+\.?|localhost|\[[\da-f:.]+\])(?::\d{1,5})?$/i; + +/** + * A raw `?`/`#` before the authority `@` begins public query or fragment text only when the text + * before it is host-shaped **and** an `=` assignment follows it, as in + * `https://host.example?contact=a@b.example`. Anything else may be a malformed username or password + * and is masked: `https://us?er:TOKEN@h/`, `http://#TOKEN@h`, `https://my.user:1234?TOKEN@h/`. A host + * fragment without an assignment (`https://host.example#sec@x`) is masked too, failing closed. Core's + * journal projection applies the same predicate (`packages/core/src/tools/bounding.ts`). + */ +function isPublicQueryContact(userinfo: string, query: number): boolean { + return HOST_SHAPED_AUTHORITY.test(userinfo.slice(0, query)) && userinfo.includes('=', query + 1); } function maskUrlUserinfo(match: string): string { @@ -101,10 +145,9 @@ function maskUrlUserinfo(match: string): string { const at = authorityAt >= 0 ? authorityAt : authority.length - 1; const userinfo = authority.slice(0, at); const query = userinfo.search(/[?#]/); - const colon = userinfo.indexOf(':'); - // Raw ?/# may occur inside a malformed password. Preserve host query/fragment assignments, - // including hosts with ports, rather than interpreting their contact values as credentials. - if (query >= 0 && (colon < 0 || colon > query || isPublicHostPort(userinfo.slice(0, query)))) + if (query >= 0 && isPublicQueryContact(userinfo, query)) return match; + // A connection string's colonless userinfo names a user (`ssh://git@host`), not a credential. + if (!userinfo.includes(':') && !COLONLESS_USERINFO_SCHEME.test(match.slice(0, start - 3))) return match; return `${match.slice(0, start)}[REDACTED]${authority.slice(at)}`; } @@ -119,9 +162,7 @@ function applyRule(text: string, rule: SecretRule): string { /** * Probe LF/TAB joins at eligible ORIGINAL starts before a greedy match can hide a later boundary. * All boundary-bearing rules use this scanner and their canonical matcher (or finite header-label - * override). A rule that has a `joinedPattern` uses it as written; any other rule MUST begin its pattern - * with `\b`, which is stripped here (deleting an explicit `joinedPattern` from a rule without `\b` - * throws on any LF/TAB input that no earlier rule has already redacted, which the suite catches at once). + * override), compiled once into `SPLIT_PROBES` (`splitProbe` rejects a rule with no joined form). * Prefixes start with ASCII letters; * that prefilter skips only impossible starts. * Keys have one token run after a bounded prefix/minimum. Header labels are bounded and their @@ -131,17 +172,18 @@ function applyRule(text: string, rule: SecretRule): string { * Opaque Bearer/Basic values and literal query names keep their established delimiter policy. */ function hasSplitCredential(text: string, joined: string, rule: SecretRule): boolean { - if (rule.boundary === undefined) return false; - const canonical = rule.joinedPattern ?? rule.pattern; - const source = rule.joinedPattern === undefined ? canonical.source.slice(2) : canonical.source; + const probe = SPLIT_PROBES.get(rule); + if (rule.boundary === undefined || probe === undefined) return false; // Absence under the same boundary-free canonical matcher proves there is no joined // candidate. Presence still requires the original-boundary and split checks below. // Native scanning avoids a JS/coverage branch for every byte of unrelated large text. - if (!new RegExp(source, canonical.flags).test(joined)) return false; - const pattern = new RegExp(source, canonical.flags.replace('g', 'y')); - // Only a rule whose boundary rejects ASCII word characters may skip word-preceded starts. `'a'` stands - // for all of them: every current boundary class rejects all ASCII word characters or none of them. - const wordBlocksStart = rule.boundary.test('a'); + if (!probe.preflight.test(joined)) return false; + const pattern = probe.sticky; + // Only a rule whose boundary rejects every ASCII word character may skip word-preceded starts; + // `splitProbe` refused any boundary that treats some word characters differently from others. + const wordBlocksStart = probe.wordBlocksStart; + const anchor = probe.anchor; + let anchorAt = -1; let originalOffset = 0; let joinedOffset = 0; while (originalOffset < text.length) { @@ -150,13 +192,22 @@ function hasSplitCredential(text: string, joined: string, rule: SecretRule): boo originalOffset += 1; continue; } + // An anchored rule (the URL rule's `://`) can match only within its window before the next + // anchor. Past the last anchor no match can start, and letter-heavy text far from any anchor is + // skipped without a RegExp attempt per character. + if (anchor !== undefined && anchorAt < joinedOffset) { + anchorAt = joined.indexOf(anchor.text, joinedOffset); + if (anchorAt < 0) return false; + } // The key and header boundary classes reject ASCII word characters. Skip those impossible // starts without allocating/testing RegExp objects for every admitted-size character; // the rule's own boundary and canonical matcher still decide every eligible start. The URL // rule blocks no start, so a glued scheme (`xhttps://`) is examined like any other. const code = text.charCodeAt(originalOffset); const previous = text.charCodeAt(originalOffset - 1); - const letter = (code >= 65 && code <= 90) || (code >= 97 && code <= 122); + const letter = + ((code >= 65 && code <= 90) || (code >= 97 && code <= 122)) && + (anchor === undefined || anchorAt - joinedOffset <= anchor.window); const previousWord = wordBlocksStart && ((previous >= 48 && previous <= 57) || diff --git a/packages/llm/src/split-credential-probe.test.ts b/packages/llm/src/split-credential-probe.test.ts new file mode 100644 index 00000000..5333ce4a --- /dev/null +++ b/packages/llm/src/split-credential-probe.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'vitest'; + +import { SplitProbeInvariantError, splitProbe } from './split-credential-probe.js'; + +function invariantCode(rule: Parameters[0]): string | undefined { + try { + splitProbe(rule); + return undefined; + } catch (error) { + if (error instanceof SplitProbeInvariantError) return error.code; + throw error; + } +} + +describe('split-credential scanner invariants', () => { + it('refuses a boundary-bearing rule with neither a joined pattern nor a \\b-prefixed source', () => { + expect(invariantCode({ pattern: /token-[a-z]+/g, boundary: /\w/ })).toBe( + 'joined_form_required', + ); + expect(invariantCode({ pattern: /\btoken-[a-z]+/g, boundary: /\w/ })).toBeUndefined(); + expect( + invariantCode({ pattern: /token-[a-z]+/g, joinedPattern: /token-/g, boundary: /\w/ }), + ).toBeUndefined(); + }); + + it('refuses a boundary that treats some ASCII word characters differently from others', () => { + for (const boundary of [/[a-z]/, /[A-Za-z]/, /\d/, /_/]) { + expect(invariantCode({ pattern: /\bx/g, boundary })).toBe('mixed_word_boundary'); + } + for (const boundary of [/\w/, /[A-Za-z0-9_-]/, /(?!)/]) { + expect(invariantCode({ pattern: /\bx/g, boundary })).toBeUndefined(); + } + }); + + it('reports the refusal as a typed error with a fixed message', () => { + expect(() => splitProbe({ pattern: /x/g, boundary: /\w/ })).toThrow( + new SplitProbeInvariantError('joined_form_required'), + ); + const error = new SplitProbeInvariantError('mixed_word_boundary'); + expect(error.name).toBe('SplitProbeInvariantError'); + expect(error.message).toBe( + 'scrub rule violates the split-credential scanner invariant: mixed_word_boundary', + ); + }); + + it('strips only the \\b of a source without a joined pattern', () => { + const probe = splitProbe({ pattern: /\bsk-[\w-]+/g, boundary: /\w/ }); + expect(probe.preflight.source).toBe('sk-[\\w-]+'); + expect(probe.sticky.source).toBe('sk-[\\w-]+'); + const joined = splitProbe({ pattern: /x/gi, joinedPattern: /\bkey:/gi, boundary: /\w/ }); + expect(joined.preflight.source).toBe('\\bkey:'); + }); + + it('compiles a stateless preflight and a sticky scanner that keep the other flags', () => { + const probe = splitProbe({ pattern: /x/gi, joinedPattern: /api-key:/gi, boundary: /\w/ }); + expect(probe.preflight.flags).toBe('i'); + expect(probe.sticky.flags).toBe('iy'); + // A global preflight would carry lastIndex between diagnostics and miss the second match. + expect(probe.preflight.test('API-KEY:')).toBe(true); + expect(probe.preflight.test('API-KEY:')).toBe(true); + }); + + it('derives the word-start skip from the boundary and carries the anchor through', () => { + expect(splitProbe({ pattern: /\bx/g, boundary: /\w/ }).wordBlocksStart).toBe(true); + const anchor = { text: '://', window: 32 }; + const url = splitProbe({ pattern: /x/g, joinedPattern: /x/g, boundary: /(?!)/, anchor }); + expect(url.wordBlocksStart).toBe(false); + expect(url.anchor).toBe(anchor); + }); +}); diff --git a/packages/llm/src/split-credential-probe.ts b/packages/llm/src/split-credential-probe.ts new file mode 100644 index 00000000..35c44f2b --- /dev/null +++ b/packages/llm/src/split-credential-probe.ts @@ -0,0 +1,78 @@ +/** + * Package-internal support for `scrub-secrets.ts`'s LF/TAB split-credential scanner. It is not a + * package export: the scanner's static rules are compiled once, at module load, into the probes this + * file describes, and a rule that breaks the scanner's two hidden assumptions fails there — with a + * typed error — instead of throwing a `SyntaxError` from inside `makeLlmError` on a later diagnostic. + * It imports nothing, so the pure `@relavium/llm/scrub-secrets` leaf still reaches only `terminal-safe` + * beyond its own package. + */ + +export type SplitProbeInvariantCode = 'joined_form_required' | 'mixed_word_boundary'; + +/** + * A static scrub rule violates a split-credential scanner invariant. A programming error, never + * input-driven, and not an exported error family (like `UsageCoverageInvariantError`). + */ +export class SplitProbeInvariantError extends Error { + readonly code: SplitProbeInvariantCode; + + constructor(code: SplitProbeInvariantCode) { + super(`scrub rule violates the split-credential scanner invariant: ${code}`); + this.name = 'SplitProbeInvariantError'; + this.code = code; + } +} + +/** A literal that every scanner match must reach within `window` joined characters of its start. */ +export interface SplitProbeAnchor { + readonly text: string; + readonly window: number; +} + +export interface SplitProbeRule { + readonly pattern: RegExp; + readonly joinedPattern?: RegExp; + readonly boundary: RegExp; + readonly anchor?: SplitProbeAnchor; +} + +export interface SplitProbe { + /** Non-global, so `test` carries no `lastIndex` state from one diagnostic to the next. */ + readonly preflight: RegExp; + /** Sticky; every use sets `lastIndex` first. */ + readonly sticky: RegExp; + /** The rule's boundary rejects every ASCII word character, so word-preceded starts can be skipped. */ + readonly wordBlocksStart: boolean; + readonly anchor: SplitProbeAnchor | undefined; +} + +const ASCII_WORD_CHARACTERS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_'; + +/** + * Compile one boundary-bearing rule. Two invariants are checked rather than assumed: + * + * - **A joined form exists.** The scanner probes a boundary-free form of the rule. A rule supplies + * it as `joinedPattern`, or its `pattern` begins with `\b`, which is stripped. Any other pattern + * would lose two real characters and become an invalid expression. + * - **The boundary treats every ASCII word character alike.** The scanner skips word-preceded starts + * as a block when the boundary rejects word characters; a boundary that rejected only some of them + * would let the skip hide an eligible start. + */ +export function splitProbe(rule: SplitProbeRule): SplitProbe { + const canonical = rule.joinedPattern ?? rule.pattern; + if (rule.joinedPattern === undefined && !canonical.source.startsWith(String.raw`\b`)) { + throw new SplitProbeInvariantError('joined_form_required'); + } + const verdicts = new Set( + Array.from(ASCII_WORD_CHARACTERS, (character) => rule.boundary.test(character)), + ); + if (verdicts.size !== 1) throw new SplitProbeInvariantError('mixed_word_boundary'); + const source = rule.joinedPattern === undefined ? canonical.source.slice(2) : canonical.source; + const flags = canonical.flags.replace('g', ''); + return { + preflight: new RegExp(source, flags), + sticky: new RegExp(source, `${flags}y`), + wordBlocksStart: verdicts.has(true), + anchor: rule.anchor, + }; +} diff --git a/tools/causal-controls/provenance.test.mjs b/tools/causal-controls/provenance.test.mjs index 8b553a68..366dfc17 100644 --- a/tools/causal-controls/provenance.test.mjs +++ b/tools/causal-controls/provenance.test.mjs @@ -267,10 +267,10 @@ test('every declared filesystem and secrets anchor matches current production so }); test('secrets workload corrections retain independent complete-count and reader-witness pins', () => { - assert.equal(secretsControls.length, 57); + assert.equal(secretsControls.length, 66); assert.equal( secretsControls.reduce((count, control) => count + 1 + (control.witnesses ?? []).length, 0), - 336, + 416, ); for (const [id, file, expectedTests] of [ ['run-default-secret-reader', 'apps/cli/src/commands/run.test.ts', 84], @@ -309,7 +309,9 @@ test('secrets workload corrections retain independent complete-count and reader- 'core-raw-password-userinfo', 'core-joined-original-boundary-recovery', 'core-header-original-boundary-recovery', - ].map((id) => [id, ['packages/core/src/tools/bounding.test.ts'], 142]), + 'core-userinfo-contact-assignment', + 'core-userinfo-contact-host', + ].map((id) => [id, ['packages/core/src/tools/bounding.test.ts'], 150]), ...[ 'diagnostic-joined-before-masking', 'diagnostic-joined-original-boundary', @@ -317,7 +319,14 @@ test('secrets workload corrections retain independent complete-count and reader- 'diagnostic-raw-password-userinfo', 'diagnostic-joined-original-boundary-recovery', 'diagnostic-header-original-boundary-recovery', - ].map((id) => [id, ['packages/llm/src/scrub-secrets.test.ts'], 111]), + 'diagnostic-header-scheme-label', + 'diagnostic-userinfo-contact-assignment', + 'diagnostic-userinfo-contact-host', + 'diagnostic-connection-scheme-userinfo', + 'diagnostic-joined-url-userinfo', + 'diagnostic-url-glued-scheme', + 'diagnostic-url-scheme-window', + ].map((id) => [id, ['packages/llm/src/scrub-secrets.test.ts'], 174]), ...['diagnostic-inline-projection-order', 'diagnostic-original-control-boundary'].map((id) => [ id, ['apps/cli/src/render/sanitize.test.ts', 'apps/cli/src/process/render-error.test.ts'], diff --git a/tools/secrets-controls/README.md b/tools/secrets-controls/README.md index aa975388..74eaab7d 100644 --- a/tools/secrets-controls/README.md +++ b/tools/secrets-controls/README.md @@ -4,7 +4,7 @@ Step 4 first-run input matrix, Step 5 snapshot/export, Step 7 provider-header and PR #95 B2 redaction controls on physical source copies. `--mechanism ` supports diagnosis only; selected runs retain `complete: false`. The current manifest -declares **57 independent controls and 336 required named assertion witnesses**. +declares **66 independent controls and 416 required named assertion witnesses**. These counts describe the workload; a declaration is not evidence that its removal passed. Actual fixed/removal/restored results belong to the coordinator's summary. @@ -177,8 +177,8 @@ modern `sk-` body characters, joined-span application, the original-text left-bo check, split scheme labels, split API-header labels and raw `?`/`#` password recognition independently. Four LLM controls remove the before-masking LF/TAB probe, its original left-boundary check, split API-header recognition and raw-password recognition. -Every core phase executes the complete 142-case bounding file; every new LLM phase -executes the complete 111-case scrubber file. Independent ordinary tooling tests pin +Every core phase executes the complete 150-case bounding file; every new LLM phase +executes the complete 174-case scrubber file. Independent ordinary tooling tests pin these inventories, the 278-case overlap cohort, 180-case CLI diagnostic cohort and 18-case command-export cohort. Actual collection still has to match in every phase. @@ -259,18 +259,31 @@ previous-word prefilter and the rule's own boundary predicate; retaining either would leave the tested policy active. Header/key recovery removals still change their original-rule boundary only, and all removals retain the same canonical matchers and bounded raw/joined standalone visitors. Current collection is 25 -complete files / 1,343 cases, with each control's exact workload and named target +complete files / 1,414 cases, with each control's exact workload and named target checked separately. These are updated declarations and author checks, not new physical causal captures or a clean review round. R3 is preserved as NONCLEAN; later corrective acceptance belongs to the owning phase register. Prior selected `7e9096a8` captures retain their older source/workload attribution. -The current 57-control / 171-phase full declaration projects 1,033 retained files -(57 × 18 + 7) under the unchanged 2,000-file / 64-MiB export bounds. This is a capacity +The current 66-control / 198-phase full declaration projects 1,195 retained files +(66 × 18 + 7) under the unchanged 2,000-file / 64-MiB export bounds. This is a capacity calculation, not a measured full capture. Historical 41-control captures retain their original source and cohort attribution; selected B2 or original-boundary recovery captures are not full-family acceptance. +Wave 3's PR95 increment adds nine controls and 80 named assertions, retaining all 57 +prior identities and witnesses. The LLM header control restores the old header value +class, so a value that begins with a `Bearer`/`Basic` label leaks its token again. Two +LLM and two core controls each drop one half of the raw `?`/`#` contact predicate (the +host-shaped prefix, or the `=` assignment that must follow it). The connection-scheme +control restores the original six-scheme list (`sftp` is no witness: the old list still +masks it by its `ftp` suffix). Three controls cut the URL rule's split-scanner support: +its joined form, its no-boundary start (a glued scheme) and its `://` window (`ws` alone +fits a two-character window, so it is no witness). Each of the nine ran in diagnosis mode +with its fixed and restored phases green and every declared witness failing on its marker; +that is author evidence, not a full-family capture. The eight-MiB scrubber cases now carry a 15-second +case deadline instead of a wall-clock assertion, inside the phases' `CAUSAL_TIMEOUT_BUDGET`. + The thin entry point uses the same coordinator, outcome and bounded process join as the filesystem family in [causal-controls](../causal-controls/README.md). The provider-header diagnostic control pins the store's fixed throwable message diff --git a/tools/secrets-controls/mutations.mjs b/tools/secrets-controls/mutations.mjs index 81f90dad..236ae471 100644 --- a/tools/secrets-controls/mutations.mjs +++ b/tools/secrets-controls/mutations.mjs @@ -66,6 +66,61 @@ const rawPasswordAssignments = ['?', '#'].map( (separator) => `preserves public ${separator} tails after userinfo and masks raw password assignments`, ); +// PR95: the extended URL scheme list, the header scheme label and the raw ?/# contact predicate. +const urlSchemes = String.raw`https?|wss?|ftps?|postgres(?:ql)?|mysql|mariadb|rediss?|mongodb(?:\+srv)?|amqps?|ssh|sftp|ldaps?|smtps?`; +const colonlessSchemes = ['http', 'https', 'ws', 'wss', 'ftp', 'ftps']; +const connectionSchemes = [ + 'postgres', + 'postgresql', + 'mysql', + 'mariadb', + 'redis', + 'rediss', + 'mongodb', + 'mongodb+srv', + 'amqp', + 'amqps', + 'ssh', + 'sftp', + 'ldap', + 'ldaps', + 'smtp', + 'smtps', +]; +const headerSchemeLabelMarker = 'SECRETS_B2_HEADER_SCHEME_LABEL'; +const headerSchemeLabelCases = [ + ...['api-key', 'x-api-key', 'x-goog-api-key', 'X-Api-Key'].map( + (name) => `masks the token after a scheme label in the ${name} header`, + ), + 'masks every LF/TAB split position inside a scheme label after a header', + 'masks the next token, failing closed, when a scheme label ends the line', +]; +const userinfoContactMarker = 'SECRETS_B2_USERINFO_CONTACT'; +const coreUserinfoContactMarker = 'SECRETS_B2_CORE_USERINFO_CONTACT'; +const contactAssignmentCases = (secondInput) => + ['https://my.user:1234?TOKEN@h.example/', secondInput].map( + (input) => `masks a host-shaped prefix that no assignment follows: ${input}`, + ); +const contactHostCase = + 'masks userinfo whose raw ?/# follows no host-shaped prefix: https://us?er=1:TOKEN@h.example/'; +const contactAssignment = String.raw` return HOST_SHAPED_AUTHORITY.test(userinfo.slice(0, query)) && userinfo.includes('=', query + 1);`; +const connectionSchemeMarker = 'SECRETS_B2_CONNECTION_SCHEME'; +// sftp also matches the original list by its ftp suffix, so the old list still masks it. +const connectionSchemeCases = connectionSchemes + .filter((scheme) => scheme !== 'sftp') + .map((scheme) => `masks colon-bearing ${scheme} userinfo and keeps a bare user`); +const urlJoinedMarker = 'SECRETS_B2_URL_JOINED'; +const urlJoinedCases = (schemes) => [ + ...schemes.map((scheme) => `masks a split ${scheme} credential around an inner masked span`), + ...['?', '#'].map( + (raw) => `masks a split credential holding a raw ${raw} and an inner masked span`, + ), +]; +const urlGluedMarker = 'SECRETS_B2_URL_GLUED_SCHEME'; +const urlGluedCases = ['\n', '\t'].map( + (separator) => + `masks a glued scheme whose split credential holds a raw @ (${JSON.stringify(separator)})`, +); const originalBoundaryMarker = 'SECRETS_B2_ORIGINAL_BOUNDARY'; const originalBoundaryCases = ['sk-proj-', 'AIza'].flatMap((prefix) => ['\n', '\t'].map( @@ -221,7 +276,7 @@ function coreProjectionControl(id, before, after, targets, marker = coreProjecti before, after, coreProjectionCases, - 142, + 150, targets[0], marker, targets.slice(1).map((target) => ({ target, marker })), @@ -234,7 +289,7 @@ function diagnosticControl(id, before, after, targets, marker = diagnosticMarker before, after, ['packages/llm/src/scrub-secrets.test.ts'], - 111, + 174, targets[0], marker, targets.slice(1).map((target) => ({ target, marker })), @@ -967,10 +1022,87 @@ export const mutations = [ ), diagnosticControl( 'diagnostic-raw-password-userinfo', - String.raw`const URL_USERINFO_PATTERN = /(?:https?|wss?|ftps?):\/\/[^/\s]*@/gi;`, - String.raw`const URL_USERINFO_PATTERN = /(?:https?|wss?|ftps?):\/\/[^/\s?#]*@/gi;`, + String.raw` /(?:${urlSchemes}):\/\/[^/\s]*@/gi;`, + String.raw` /(?:${urlSchemes}):\/\/[^/\s?#]*@/gi;`, [...diagnosticPasswords, ...rawPasswordAssignments], ), + // PR95: a header value's leading Bearer/Basic label is masked with its token. + diagnosticControl( + 'diagnostic-header-scheme-label', + String.raw` /(^|[^A-Za-z0-9_-])((?:x-)?api-key|x-goog-api-key)(["']?\s*:\s*["']?)(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r\s+|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c\s+)?[^\s"',;}]+/gi,`, + String.raw` /(^|[^A-Za-z0-9_-])((?:x-)?api-key|x-goog-api-key)(["']?\s*:\s*["']?)[^\s"',;}]+/gi,`, + headerSchemeLabelCases, + headerSchemeLabelMarker, + ), + // PR95: raw ?/# userinfo is public only after a host-shaped prefix AND an assignment. + diagnosticControl( + 'diagnostic-userinfo-contact-assignment', + contactAssignment, + String.raw` return HOST_SHAPED_AUTHORITY.test(userinfo.slice(0, query));`, + contactAssignmentCases('https://host.example#sec@x.example'), + userinfoContactMarker, + ), + control( + 'diagnostic-userinfo-contact-host', + scrubSecretsLeaf, + contactAssignment, + String.raw` return userinfo.includes('=', query + 1);`, + ['packages/llm/src/scrub-secrets.test.ts'], + 174, + contactHostCase, + userinfoContactMarker, + rawPasswordAssignments.map((target) => ({ target, marker: diagnosticMarker })), + ), + coreProjectionControl( + 'core-userinfo-contact-assignment', + contactAssignment, + String.raw` return HOST_SHAPED_AUTHORITY.test(userinfo.slice(0, query));`, + contactAssignmentCases('https://provider.example#section@x.example'), + coreUserinfoContactMarker, + ), + control( + 'core-userinfo-contact-host', + coreProjection, + contactAssignment, + String.raw` return userinfo.includes('=', query + 1);`, + coreProjectionCases, + 150, + contactHostCase, + coreUserinfoContactMarker, + rawPasswordAssignments.map((target) => ({ target, marker: coreProjectionMarker })), + ), + // PR95: connection-string schemes join the finite URL list. + diagnosticControl( + 'diagnostic-connection-scheme-userinfo', + String.raw` /(?:${urlSchemes}):\/\/[^/\s]*@/gi;`, + String.raw` /(?:https?|wss?|ftps?):\/\/[^/\s]*@/gi;`, + connectionSchemeCases, + connectionSchemeMarker, + ), + // PR95: only the joined scanner sees a split credential around an inner masked span. + diagnosticControl( + 'diagnostic-joined-url-userinfo', + ' joinedPattern: URL_USERINFO_PATTERN,', + ' joinedPattern: /a^/g,', + urlJoinedCases([...colonlessSchemes, ...connectionSchemes]), + urlJoinedMarker, + ), + // PR95: the URL rule blocks no start, so a glued scheme is scanned like any other. + diagnosticControl( + 'diagnostic-url-glued-scheme', + ' boundary: NO_BOUNDARY,', + String.raw` boundary: /\w/,`, + urlGluedCases, + urlGluedMarker, + ), + // PR95: the `://` prefilter window must cover every listed scheme (`ws` alone fits in two). + diagnosticControl( + 'diagnostic-url-scheme-window', + " anchor: { text: '://', window: URL_SCHEME_WINDOW },", + " anchor: { text: '://', window: 2 },", + urlJoinedCases([...colonlessSchemes, ...connectionSchemes].filter((scheme) => scheme !== 'ws')), + urlJoinedMarker, + ), // R1-LN-1: recover a valid original start hidden inside a rejected greedy joined candidate. coreProjectionControl( 'core-joined-original-boundary-recovery', From ef3ded0d4a01fa1adf90f91e1f15188e079e2258 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Tue, 20 Oct 2026 01:07:41 +0300 Subject: [PATCH 10/97] fix(llm): address the first PR95 review round Round 1 of the PR95 review was NONCLEAN in all five lenses; every verified finding is fixed here, in llm, core's twin predicate, the CLI tests, the secrets causal controls (repo tooling) and the docs. - The host-shaped ?/# predicate ran an unbounded dotted run through V8's RegExp backtrack stack: an admitted-size prefix made scrubSecrets, makeLlmError and core's redactSecretShapedText/boundForModel throw RangeError. Both maskers now refuse a prefix longer than 260 characters (a DNS name plus a port) before the RegExp, which fails closed. - A header value with stacked or quoted labels (Bearer Bearer , Bearer "") leaked the token. Up to eight labels and an optional quote now precede the token, bounded so no label run can exhaust the stack, and the standalone Bearer/Basic rules accept the same stacked labels. - Already-masked userinfo is a fixed point (an exactly masked password under a connection-string scheme, or every part masked or empty under any scheme), so a templated postgres://app:{{secrets.password}}@db no longer raises the authored-content notice on every run. - New tests pin the host-shaped anchors and port bound, multi-character label whitespace, stacking, the length bound at admitted size and the fixed point. Six new causal controls (72 in all, 430 named witnesses) and the snapshot detector's 11-case workload follow. - error-handling.md now states which rules probe LF/TAB joins, how the ?/# rule combines with the scheme rule, stacked labels, fixed points and the unlisted-scheme and whitespace-delimited limits; built-in-tools.md lists every core difference; commands.md, current.md and the rulings record (per-increment security sittings) are corrected. Refs: ADR-0113 Co-Authored-By: Claude Opus 5.5 --- .../cli/src/authoring/snapshot-notice.test.ts | 21 ++++ docs/reference/cli/commands.md | 4 +- docs/reference/shared-core/built-in-tools.md | 9 +- docs/roadmap/current.md | 4 +- .../phase-2.5.5-hardening-and-remediation.md | 2 +- docs/standards/error-handling.md | 59 ++++++---- packages/core/src/tools/bounding.test.ts | 29 +++++ packages/core/src/tools/bounding.ts | 12 +- packages/llm/src/scrub-secrets.test.ts | 103 ++++++++++++++++++ packages/llm/src/scrub-secrets.ts | 51 +++++++-- tools/causal-controls/provenance.test.mjs | 14 ++- tools/secrets-controls/README.md | 20 +++- tools/secrets-controls/mutations.mjs | 72 ++++++++++-- 13 files changed, 338 insertions(+), 62 deletions(-) diff --git a/apps/cli/src/authoring/snapshot-notice.test.ts b/apps/cli/src/authoring/snapshot-notice.test.ts index 35cf71dc..df5e6393 100644 --- a/apps/cli/src/authoring/snapshot-notice.test.ts +++ b/apps/cli/src/authoring/snapshot-notice.test.ts @@ -48,6 +48,27 @@ describe('authored snapshot credential notice', () => { expect(err()).toBe(''); }); + it('keeps placeholder connection-string passwords quiet and reports a literal http(s) username', () => { + const quiet = captureIo(); + warnSnapshotLiterals( + { + env: { + DATABASE_URL: 'postgres://app:{{secrets.db_password}}@db.example/app', + CACHE_URL: 'redis://:{{secrets.cache_password}}@cache.example:6379', + }, + }, + quiet.io, + ); + expect(quiet.err()).toBe(''); + // http(s)/ws(s)/ftp(s) mask any userinfo, so a literal username beside a placeholder still counts. + const reported = captureIo(); + warnSnapshotLiterals( + { env: { REPOSITORY: 'https://app:{{secrets.token}}@git.example/repo.git' } }, + reported.io, + ); + expect(reported.err()).toContain('1 string(s) matching known credential shapes'); + }); + it('does not exempt credential literals beside placeholders or inside filter arguments', () => { const literal = 'sk-templateSyntheticCredential123456789'; const { io, err } = captureIo(); diff --git a/docs/reference/cli/commands.md b/docs/reference/cli/commands.md index 7e9a1acd..b4453f1c 100644 --- a/docs/reference/cli/commands.md +++ b/docs/reference/cli/commands.md @@ -232,9 +232,9 @@ Both are **surface-agnostic** git-native YAML operations — they read/write `.r `create` and `import` emit these notices only after their YAML file is written. A refused write emits no authored-content notice or success output. -The review visits at most 4,096 nodes through depth 64, checks strings of at most 16,384 UTF-16 code units and at most 262,144 checked code units in total, including property names. It reports only a count of strings matching the finite detector: prefixed OpenAI/Anthropic or Google keys, HTTP(S)/WS(S)/FTP(S) URL userinfo, exact `api-key`/`x-api-key`/`x-goog-api-key` headers, Bearer/Basic credentials and literal secret query names. It does not classify every credential format or distinguish arbitrary opaque credential fragments from following diagnostic prose. Over-budget strings or structure, repeated objects and unsupported object shapes produce a separate fixed incomplete-review notice. Object-key enumeration relies on the existing parser source ceiling; the node limit is not an independent bound on that enumeration. Neither notice includes values or arbitrary property names. +The review visits at most 4,096 nodes through depth 64, checks strings of at most 16,384 UTF-16 code units and at most 262,144 checked code units in total, including property names. It reports only a count of strings matching the finite detector, the [diagnostic secret backstop](../../standards/error-handling.md#diagnostic-secret-backstop): prefixed OpenAI/Anthropic or Google keys, URL userinfo (any userinfo for HTTP(S)/WS(S)/FTP(S), colon-bearing userinfo for the listed connection-string schemes), exact `api-key`/`x-api-key`/`x-goog-api-key` headers, Bearer/Basic credentials and literal secret query names. It does not classify every credential format or distinguish arbitrary opaque credential fragments from following diagnostic prose. Over-budget strings or structure, repeated objects and unsupported object shapes produce a separate fixed incomplete-review notice. Object-key enumeration relies on the existing parser source ceiling; the node limit is not an independent bound on that enumeration. Neither notice includes values or arbitrary property names. -For this detector probe only, a recognised bare template reference without a path or filter is projected to an already masked span, provided its raw reference contains no recognised credential shape. This keeps `Bearer {{secrets.token}}` and equivalent query placeholders quiet. Adjacent literals, credential-shaped identifiers, paths, filter arguments and unknown or malformed references retain their evidence. This projection neither resolves references nor changes authored content. +For this detector probe only, a recognised bare template reference without a path or filter is projected to an already masked span, provided its raw reference contains no recognised credential shape. This keeps `Bearer {{secrets.token}}`, equivalent query placeholders and a connection string whose password is a placeholder (`postgres://app:{{secrets.db_password}}@db`) quiet. An HTTP(S)/WS(S)/FTP(S) URL with a literal username beside a placeholder password still counts, because those schemes mask any userinfo. Adjacent literals, credential-shaped identifiers, paths, filter arguments and unknown or malformed references retain their evidence. This projection neither resolves references nor changes authored content. ### `relavium logs ` diff --git a/docs/reference/shared-core/built-in-tools.md b/docs/reference/shared-core/built-in-tools.md index 7a7405fa..23e31eae 100644 --- a/docs/reference/shared-core/built-in-tools.md +++ b/docs/reference/shared-core/built-in-tools.md @@ -96,9 +96,12 @@ existing content-sensitive handling. A raw `?` or `#` before a URL authority's `@` begins public query or fragment text only when the text before it is host-shaped (a dotted name or IPv4, `localhost` or a bracketed IP literal, with an optional port) **and** an `=` assignment follows it; otherwise the userinfo is masked. -This is the same predicate as the [LLM diagnostic backstop](../../standards/error-handling.md#diagnostic-secret-backstop). -Two projection-only differences remain by design: `]` ends an authority, and a userinfo -with no password that is shorter than 16 characters is kept as a vanity handle. +This is the same predicate as the [LLM diagnostic backstop](../../standards/error-handling.md#diagnostic-secret-backstop), +including its 260-character bound on the host-shaped prefix. The projection otherwise differs from +that backstop by design: its authority also ends at `"`, `'`, `,`, `}` and `]`; it recognizes any +scheme of up to 32 characters rather than a finite list; a userinfo with no password is kept as a +vanity handle below 16 characters and masked from 16 on, under every scheme; and it does not treat +an already-masked password as masked. LF/TAB inside a recognized token-shaped credential extends the masked original span. An ambiguous token-alphabet continuation after LF/TAB is conservatively masked with diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index a64498b0..4bb4ba4f 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -1154,8 +1154,8 @@ the follow-up corrections are on `development`, not yet on `main`. Whole-wave ac Decompose, guard and normalize every hot file a 2.6 workstream is about to land on. **The multi-file queue is Step 1 and runs first.** The approved -[nineteen-increment execution plan](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) -owns the full order and review gates: finish each increment and its fresh reviews +[nineteen-step execution plan](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments), +recorded as 29 increments since 2026-10-19, owns the full order and review gates: finish each increment and its fresh reviews before advancing; no parallel edits of shared hot files. The groups below retain the original scope inventory and within-file dependency diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 92e0f1e2..38fc5160 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -9393,7 +9393,7 @@ plan amendment (row PR95) and the recorded splits; ID coverage is unchanged apar | Area | Ruling | | --- | --- | -| Acceptance | Each increment keeps the existing rule: implement, author gates, commit, then NEW independent complete cumulative review rounds until two consecutive rounds are clean after the last material fix. Comment- or documentation-only nits do not reset the count. Security-touching increments add a separate scoped sitting, and whole-step sittings stay where their rows require them. | +| Acceptance | Each increment keeps the existing rule: implement, author gates, commit, then NEW independent complete cumulative review rounds until two consecutive rounds are clean after the last material fix. Comment- or documentation-only nits do not reset the count. Security-touching increments add a separate scoped sitting: PR95, 9a.1, 9a.2, 9b, 9c, 9d (whole Step 9 custody), 10a, 10c, 11b, 13c, 14b, 15c, 16a, 16b, 16c (whole Step 16), 18 and 19b (whole wave). | | Publication | `development` is pushed after each accepted increment, so its actual CI (including the four same-tarball platform consumers) is observed per increment. The Wave 3 PR follows Step 19. | | Plan shape | PR95 + 9a.1, 9a.2, 9b, 9c, 9d + 10a–c + 11a–b + 12a–b + 13a–c + 14a–b + 15a–c + 16a–c + 17a–b + 18 + 19a–b (29 increments). Each split is recorded before its step starts. | | Reassignments | G54 moves from row 17 to 11a; #311's implementation moves to 10b and its certification stays in 17a; G36/G46 are re-scoped in 10a to the branch that replaces the raw-stack branch and are certified in 17b; promotion of the Steps 6–10 commands.md sections moves from Step 19 to 10c. | diff --git a/docs/standards/error-handling.md b/docs/standards/error-handling.md index c7c38b4c..a4b314c0 100644 --- a/docs/standards/error-handling.md +++ b/docs/standards/error-handling.md @@ -129,31 +129,44 @@ message, and the backstop masks a finite set of secret **shapes**, not arbitrary content. - **Shapes.** OpenAI/Anthropic `sk-…` and Google `AIza…` key prefixes; a finite list of literal - secret query names; the exact `api-key`, `x-api-key` and `x-goog-api-key` header names (a - value that begins with a `Bearer`/`Basic` label is masked together with its token); and - `Bearer`/`Basic` authentication values. -- **URL userinfo.** The schemes `http(s)`, `ws(s)` and `ftp(s)` mask any userinfo. The - connection-string schemes `postgres`, `postgresql`, `mysql`, `mariadb`, `redis`, `rediss`, - `mongodb`, `mongodb+srv`, `amqp`, `amqps`, `ssh`, `sftp`, `ldap`, `ldaps`, `smtp` and `smtps` - mask colon-bearing userinfo (`user:pass@`, `:pass@`); a bare user such as `ssh://git@host` - stays visible. A raw `?` or `#` before the authority `@` begins public query or fragment - text only when the text before it is host-shaped (a dotted name or IPv4, `localhost` or a - bracketed IP literal, with an optional port) **and** an `=` assignment follows it, as in - `https://host.example?contact=a@b.example`; otherwise the userinfo is masked. Core's journal - projection applies the same `?`/`#` predicate + secret query names; the exact `api-key`, `x-api-key` and `x-goog-api-key` header names; and + `Bearer`/`Basic` authentication values. A header value that begins with up to eight + `Bearer`/`Basic` labels (a doubled `Bearer Bearer` is a common misconfiguration) is masked + together with the token that follows them, quoted or not; a standalone value accepts the same + stacked labels before its token. +- **URL userinfo.** The `?`/`#` rule decides where userinfo ends: a raw `?` or `#` before the + authority `@` begins public query or fragment text only when the text before it is host-shaped + (a dotted name or IPv4, `localhost` or a bracketed IP literal, with an optional port, at most + 260 characters in all) **and** an `=` assignment follows it, as in + `https://host.example?contact=a@b.example`. The scheme rule then decides what is masked: + `http(s)`, `ws(s)` and `ftp(s)` mask any userinfo, while the connection-string schemes + `postgres`, `postgresql`, `mysql`, `mariadb`, `redis`, `rediss`, `mongodb`, `mongodb+srv`, + `amqp`, `amqps`, `ssh`, `sftp`, `ldap`, `ldaps`, `smtp` and `smtps` mask only colon-bearing + userinfo (`user:pass@`, `:pass@`), so a bare user such as `ssh://git@host` (or a colonless + `redis://#TOKEN@host`) stays visible. Userinfo that is already masked stays as it is: an + exactly masked password under a connection-string scheme, or every part masked or empty under + any scheme, so `postgres://app:{{secrets.password}}@db` projects to a fixed point. Core's + journal projection applies the same `?`/`#` predicate ([built-in tools](../reference/shared-core/built-in-tools.md#finite-diagnostic-credential-masking)). -- **Split and normalized forms.** Each rule also probes the terminal-normalized text and the - LF/TAB-joined view at original-text boundaries; a credential revealed only by - normalization or by an LF/TAB split masks the **whole** diagnostic. +- **Split and normalized forms.** Every rule except the literal query names also probes the + terminal-normalized text; a credential revealed only by normalization masks the **whole** + diagnostic. The key prefixes, URL + userinfo and the `api-key` header label are additionally probed in the LF/TAB-joined view at + original-text boundaries, and a credential revealed only by such a split masks the whole + diagnostic too. A `Bearer`/`Basic` label may itself be split by LF/TAB. - **Documented trade-offs, all failing closed.** A scheme glued to a word character is masked in - place; an unlisted scheme ending in a listed one (`gftp://`) matches by its suffix; prose - that the whitespace-stripped residual probe glues to a path-less URL and a later `@` blanks the - whole diagnostic; a host fragment with no assignment (`https://host.example#sec@x`) and a - single-label host query are masked; and a `Bearer`/`Basic` label that ends a line takes the - next token with it. -- **Limits.** Malformed URLs (`https:/u:p@h`), a raw `/` inside a password, invisible characters - inside a scheme and opaque fragments separated by ordinary spaces are outside the finite - shapes. + place; an unlisted scheme ending in a listed one (`gftp://`, `jdbc:mysql://`) matches by its + suffix; prose that the whitespace-stripped residual probe glues to a path-less URL and a later + `@` blanks the whole diagnostic; a host fragment with no assignment (`https://host.example#sec@x`), + a single-label host query and an over-long host-shaped prefix are masked; and a + `Bearer`/`Basic` label that ends a line takes the next token with it. +- **Limits.** Userinfo under an unlisted scheme is not masked, including driver-qualified + spellings such as `postgresql+psycopg2://`. Opaque `Bearer`/`Basic` values, `api-key` header + values and literal secret query values end at any whitespace, LF and TAB included, so a + fragment after such a break stays visible, as it does after an ordinary space; a header token + that begins with `,`, `;` or `}` after a label is likewise outside the shape. Malformed URLs + (`https:/u:p@h`), a raw `/` inside a password and invisible characters inside a scheme are + outside the finite shapes. ## No silent catches diff --git a/packages/core/src/tools/bounding.test.ts b/packages/core/src/tools/bounding.test.ts index 19a34b4b..d16764a2 100644 --- a/packages/core/src/tools/bounding.test.ts +++ b/packages/core/src/tools/bounding.test.ts @@ -1209,8 +1209,37 @@ describe('PR95 raw ?/# userinfo in the journal projection', () => { 'https://provider.invalid:8443#contact=alice@example.invalid', 'https://localhost:3000?contact=alice@example.invalid', 'https://10.0.0.1#contact=alice@example.invalid', + 'https://john.doe:s3cret?x=1@h.example/', + 'https://localhostuser:pw?a=b@h.example/', + 'https://user:pw.evil.com?x=1@h.example/', + 'https://h.example:123456?x=1@h.example/', + 'https://h.example:65535?contact=alice@example.invalid', ]) { expect(redactSecretShapedText(input) === input, input).toBe(scrubSecrets(input) === input); } }); + + it.each([ + 'https://john.doe:s3cret?x=1@h.example/', + 'https://localhostuser:pw?a=b@h.example/', + 'https://user:pw.evil.com?x=1@h.example/', + 'https://h.example:123456?x=1@h.example/', + ])('masks userinfo whose prefix only resembles a host: %s', (input) => { + expect(redactSecretShapedText(input), 'SECRETS_B2_CORE_USERINFO_CONTACT').toBe( + 'https://[redacted]@h.example/', + ); + }); + + it('masks an admitted-size dotted userinfo prefix without exhausting the RegExp stack', () => { + const labels = 'a.'.repeat(Math.floor((8 * 1024 * 1024 - 256 - 64) / 2)); + const input = `upstream said https://${labels}a?x=1@h failed`; + let output = ''; + expect(() => { + output = redactSecretShapedText(input); + }, 'SECRETS_B2_CORE_USERINFO_HOST_LENGTH').not.toThrow(); + expect( + output === 'upstream said https://[redacted]@h failed', + 'SECRETS_B2_CORE_USERINFO_HOST_LENGTH', + ).toBe(true); + }, 15_000); }); diff --git a/packages/core/src/tools/bounding.ts b/packages/core/src/tools/bounding.ts index 428f7993..13c7e779 100644 --- a/packages/core/src/tools/bounding.ts +++ b/packages/core/src/tools/bounding.ts @@ -435,6 +435,9 @@ function redactOpaqueQueryValues(text: string): string { const HOST_SHAPED_AUTHORITY = /^(?:[a-z\d-]+(?:\.[a-z\d-]+)+\.?|localhost|\[[\da-f:.]+\])(?::\d{1,5})?$/i; +/** A DNS name has at most 253 characters (254 with a trailing dot), and a `:port` adds at most six. */ +const HOST_SHAPED_MAX_LENGTH = 260; + /** * One whole authority, with its last `@` separating userinfo from the host. Prefix-only * matches can mask the first password fragment and leave another `@` plus a credential @@ -443,10 +446,15 @@ const HOST_SHAPED_AUTHORITY = * begins public query/fragment text only when the text before it is host-shaped **and** an * `=` assignment follows it; anything else may be a malformed username or password. This is * the same predicate as the LLM diagnostic backstop (`packages/llm/src/scrub-secrets.ts`). - * Two core-only differences remain by design: `]` ends an authority, and a userinfo with no - * password that is shorter than 16 characters is a vanity handle under the existing non-secret policy. + * The projection differs from that backstop by design: its authority also ends at `"`, `'`, `,`, + * `}` and `]`; it recognizes any scheme of up to 32 characters rather than a finite list; a + * userinfo with no password is a vanity handle below 16 characters and masked from 16 on, under + * every scheme; and it does not treat an already-masked password as masked. */ function isPublicQueryContact(userinfo: string, query: number): boolean { + // A longer prefix cannot be a hostname, and an unbounded dotted run would exhaust V8's RegExp + // backtrack stack at admitted size; such a prefix stays userinfo, which fails closed. + if (query > HOST_SHAPED_MAX_LENGTH) return false; return HOST_SHAPED_AUTHORITY.test(userinfo.slice(0, query)) && userinfo.includes('=', query + 1); } diff --git a/packages/llm/src/scrub-secrets.test.ts b/packages/llm/src/scrub-secrets.test.ts index a8044c78..7732a55d 100644 --- a/packages/llm/src/scrub-secrets.test.ts +++ b/packages/llm/src/scrub-secrets.test.ts @@ -618,3 +618,106 @@ describe('PR95 joined-view and glued-scheme witnesses', () => { }, ); }); + +describe('PR95 review round 1: label stacking, host bounds and masked userinfo', () => { + const token = ['abcdef', 'ghijkl', '123456'].join(''); + const headerSchemeLabel = (value: unknown) => expect(value, 'SECRETS_B2_HEADER_SCHEME_LABEL'); + const userinfoContact = (value: unknown) => expect(value, 'SECRETS_B2_USERINFO_CONTACT'); + + it('masks the token after stacked or quoted scheme labels in a header value', () => { + for (const [input, expected] of [ + [`api-key: Bearer Bearer ${token}`, 'api-key: [REDACTED]'], + [`x-api-key: Basic Bearer ${token}`, 'x-api-key: [REDACTED]'], + [`{"x-api-key":"Bearer Bearer ${token}"}`, '{"x-api-key":"[REDACTED]"}'], + [`x-goog-api-key: Bearer\nBearer ${token}`, 'x-goog-api-key: [REDACTED]'], + [`api-key:Bearer B\tearer ${token}`, 'api-key:[REDACTED]'], + [`api-key: Bearer "${token}"`, 'api-key: [REDACTED]"'], + [`api-key: Basic '${token}'`, "api-key: [REDACTED]'"], + ] as const) { + const output = scrubSecrets(input); + headerSchemeLabel(output).toBe(expected); + headerSchemeLabel(output.includes(token)).toBe(false); + } + }); + + it('masks the token after more than one whitespace character following a scheme label', () => { + for (const [input, expected] of [ + [`api-key: Bearer ${token} rejected`, 'api-key: [REDACTED] rejected'], + [`x-api-key:Basic \t${token}`, 'x-api-key:[REDACTED]'], + [`api-key:Bearer \r\n${token}`, 'api-key:[REDACTED]'], + ] as const) { + const output = scrubSecrets(input); + headerSchemeLabel(output).toBe(expected); + headerSchemeLabel(output.includes(token)).toBe(false); + } + }); + + it('masks the token after stacked labels in a standalone scheme value', () => { + for (const [input, expected] of [ + [ + `Authorization: Bearer Bearer ${token} rejected`, + 'Authorization: Bearer [REDACTED] rejected', + ], + [`Authorization: Basic Basic ${token}`, 'Authorization: Basic [REDACTED]'], + [`Authorization: Bearer Basic ${token}`, 'Authorization: Bearer [REDACTED]'], + [`Authorization: Bearer B\nearer ${token}`, 'Authorization: Bearer [REDACTED]'], + ] as const) { + expect(scrubSecrets(input)).toBe(expected); + } + expect(scrubSecrets('Authorization: Bearer Bearerish rejected')).toBe( + 'Authorization: Bearer [REDACTED] rejected', + ); + }); + + it.each([ + 'https://john.doe:s3cret?x=1@h.example/', + 'https://localhostuser:pw?a=b@h.example/', + 'https://user:pw.evil.com?x=1@h.example/', + 'https://h.example:123456?x=1@h.example/', + ])('masks userinfo whose prefix only resembles a host: %s', (input) => { + userinfoContact(scrubSecrets(input)).toBe('https://[REDACTED]@h.example/'); + }); + + it('retains a public contact after a host with a five-digit port', () => { + const input = 'connect https://h.example:65535?contact=alice@example.invalid failed'; + expect(scrubSecrets(input)).toBe(input); + }); + + it( + 'masks an admitted-size dotted userinfo prefix without exhausting the RegExp stack', + () => { + const labels = 'a.'.repeat(Math.floor((8 * 1024 * 1024 - 256 - 64) / 2)); + const input = `upstream said https://${labels}a?x=1@h failed`; + let output = ''; + expect(() => { + output = scrubSecrets(input); + }, 'SECRETS_B2_USERINFO_HOST_LENGTH').not.toThrow(); + expect( + output === 'upstream said https://[REDACTED]@h failed', + 'SECRETS_B2_USERINFO_HOST_LENGTH', + ).toBe(true); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); + + it('keeps colonless connection-string userinfo visible even with a raw ?/#', () => { + for (const input of ['redis://#TOKEN@h.example', 'postgres://us?er@db.example/x']) { + expect(scrubSecrets(input)).toBe(input); + } + }); + + it('leaves already-masked userinfo as a fixed point and masks a literal around a mask', () => { + for (const input of [ + 'postgres://app:[REDACTED]@db.example/app', + 'redis://:[REDACTED]@cache.example:6379', + 'https://[REDACTED]:[REDACTED]@h.example/', + 'https://:[REDACTED]@h.example/', + ]) { + expect(scrubSecrets(input)).toBe(input); + } + expect(scrubSecrets('https://app:[REDACTED]@h.example/')).toBe('https://[REDACTED]@h.example/'); + expect(scrubSecrets('postgres://app:pa[REDACTED]ss@db.example')).toBe( + 'postgres://[REDACTED]@db.example', + ); + }); +}); diff --git a/packages/llm/src/scrub-secrets.ts b/packages/llm/src/scrub-secrets.ts index c34ff3c8..dafa43b1 100644 --- a/packages/llm/src/scrub-secrets.ts +++ b/packages/llm/src/scrub-secrets.ts @@ -84,23 +84,31 @@ const NON_QUERY_SECRET_RULES: readonly SecretRule[] = [ anchor: { text: '://', window: URL_SCHEME_WINDOW }, }, // Exact literal header names, also in quoted diagnostic objects. No arbitrary header discovery. - // A value may begin with a (possibly LF/TAB-split) Bearer/Basic label: the label and its token are + // A value may begin with up to eight (possibly LF/TAB-split) Bearer/Basic labels, as a doubled + // `Bearer Bearer` misconfiguration does, and its token may be quoted: the labels and the token are // masked together, so the later scheme-label rules cannot lose their anchor to this one. Like those - // rules, whitespace after the label may be a line break, and the next token is masked fail-closed. + // rules, whitespace after a label may be a line break, and the next token is masked fail-closed. + // The repetition is bounded so that no admitted-size label run can exhaust the RegExp stack. { pattern: - /(^|[^A-Za-z0-9_-])((?:x-)?api-key|x-goog-api-key)(["']?\s*:\s*["']?)(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r\s+|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c\s+)?[^\s"',;}]+/gi, + /(^|[^A-Za-z0-9_-])((?:x-)?api-key|x-goog-api-key)(["']?\s*:\s*["']?)(?:(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)\s+){0,8}["']?[^\s"',;}]+/gi, replacement: '$1$2$3[REDACTED]', joinedPattern: /(?:(?:x-)?api-key|x-goog-api-key)["']?\s*:/gi, boundary: /[A-Za-z0-9_-]/, }, // LF/TAB may split the finite scheme LABEL. Preserve the required label/value separator and - // ordinary following prose; opaque value fragments cannot be distinguished from that prose. + // ordinary following prose; opaque value fragments cannot be distinguished from that prose. Up to + // eight stacked labels (`Bearer Bearer `) precede the token without becoming it. { - pattern: /\bB[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r\s+\S+/gi, + pattern: + /\bB[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r(?:\s+(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)){0,8}\s+\S+/gi, replacement: 'Bearer [REDACTED]', }, - { pattern: /\bB[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c\s+\S+/gi, replacement: 'Basic [REDACTED]' }, + { + pattern: + /\bB[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c(?:\s+(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)){0,8}\s+\S+/gi, + replacement: 'Basic [REDACTED]', + }, ]; /** @@ -125,18 +133,26 @@ function normalizeSecretProbe(text: string): string { const HOST_SHAPED_AUTHORITY = /^(?:[a-z\d-]+(?:\.[a-z\d-]+)+\.?|localhost|\[[\da-f:.]+\])(?::\d{1,5})?$/i; +/** A DNS name has at most 253 characters (254 with a trailing dot), and a `:port` adds at most six. */ +const HOST_SHAPED_MAX_LENGTH = 260; + /** * A raw `?`/`#` before the authority `@` begins public query or fragment text only when the text * before it is host-shaped **and** an `=` assignment follows it, as in - * `https://host.example?contact=a@b.example`. Anything else may be a malformed username or password - * and is masked: `https://us?er:TOKEN@h/`, `http://#TOKEN@h`, `https://my.user:1234?TOKEN@h/`. A host - * fragment without an assignment (`https://host.example#sec@x`) is masked too, failing closed. Core's - * journal projection applies the same predicate (`packages/core/src/tools/bounding.ts`). + * `https://host.example?contact=a@b.example`. Otherwise the text is userinfo: `https://us?er:TOKEN@h/`, + * `http://#TOKEN@h`, `https://my.user:1234?TOKEN@h/` and a host fragment without an assignment + * (`https://host.example#sec@x`, failing closed) are masked under the rest of `maskUrlUserinfo`'s policy. + * Core's journal projection applies the same predicate (`packages/core/src/tools/bounding.ts`). */ function isPublicQueryContact(userinfo: string, query: number): boolean { + // A longer prefix cannot be a hostname, and an unbounded dotted run would exhaust V8's RegExp + // backtrack stack at admitted size; such a prefix stays userinfo, which fails closed. + if (query > HOST_SHAPED_MAX_LENGTH) return false; return HOST_SHAPED_AUTHORITY.test(userinfo.slice(0, query)) && userinfo.includes('=', query + 1); } +const MASKED = '[REDACTED]'; + function maskUrlUserinfo(match: string): string { const start = match.indexOf('://') + 3; const authority = match.slice(start); @@ -146,10 +162,21 @@ function maskUrlUserinfo(match: string): string { const userinfo = authority.slice(0, at); const query = userinfo.search(/[?#]/); if (query >= 0 && isPublicQueryContact(userinfo, query)) return match; + const colonless = COLONLESS_USERINFO_SCHEME.test(match.slice(0, start - 3)); + const colon = userinfo.indexOf(':'); // A connection string's colonless userinfo names a user (`ssh://git@host`), not a credential. - if (!userinfo.includes(':') && !COLONLESS_USERINFO_SCHEME.test(match.slice(0, start - 3))) + if (colon < 0 && !colonless) return match; + // Already-masked userinfo stays as it is: an exactly masked password under a connection-string + // scheme, whose usernames are public, or a masked or empty username beside it under any scheme. + // `postgres://app:{{secrets.password}}@db` therefore projects to a fixed point. + const username = userinfo.slice(0, Math.max(colon, 0)); + if ( + colon >= 0 && + userinfo.slice(colon + 1) === MASKED && + (!colonless || username === '' || username === MASKED) + ) return match; - return `${match.slice(0, start)}[REDACTED]${authority.slice(at)}`; + return `${match.slice(0, start)}${MASKED}${authority.slice(at)}`; } function applyRule(text: string, rule: SecretRule): string { diff --git a/tools/causal-controls/provenance.test.mjs b/tools/causal-controls/provenance.test.mjs index 366dfc17..399d543f 100644 --- a/tools/causal-controls/provenance.test.mjs +++ b/tools/causal-controls/provenance.test.mjs @@ -267,10 +267,10 @@ test('every declared filesystem and secrets anchor matches current production so }); test('secrets workload corrections retain independent complete-count and reader-witness pins', () => { - assert.equal(secretsControls.length, 66); + assert.equal(secretsControls.length, 72); assert.equal( secretsControls.reduce((count, control) => count + 1 + (control.witnesses ?? []).length, 0), - 416, + 430, ); for (const [id, file, expectedTests] of [ ['run-default-secret-reader', 'apps/cli/src/commands/run.test.ts', 84], @@ -311,7 +311,9 @@ test('secrets workload corrections retain independent complete-count and reader- 'core-header-original-boundary-recovery', 'core-userinfo-contact-assignment', 'core-userinfo-contact-host', - ].map((id) => [id, ['packages/core/src/tools/bounding.test.ts'], 150]), + 'core-userinfo-host-length', + 'core-userinfo-host-anchors', + ].map((id) => [id, ['packages/core/src/tools/bounding.test.ts'], 155]), ...[ 'diagnostic-joined-before-masking', 'diagnostic-joined-original-boundary', @@ -326,7 +328,11 @@ test('secrets workload corrections retain independent complete-count and reader- 'diagnostic-joined-url-userinfo', 'diagnostic-url-glued-scheme', 'diagnostic-url-scheme-window', - ].map((id) => [id, ['packages/llm/src/scrub-secrets.test.ts'], 174]), + 'diagnostic-header-stacked-labels', + 'diagnostic-header-label-whitespace', + 'diagnostic-userinfo-host-length', + 'diagnostic-userinfo-host-anchors', + ].map((id) => [id, ['packages/llm/src/scrub-secrets.test.ts'], 185]), ...['diagnostic-inline-projection-order', 'diagnostic-original-control-boundary'].map((id) => [ id, ['apps/cli/src/render/sanitize.test.ts', 'apps/cli/src/process/render-error.test.ts'], diff --git a/tools/secrets-controls/README.md b/tools/secrets-controls/README.md index 74eaab7d..afe2f706 100644 --- a/tools/secrets-controls/README.md +++ b/tools/secrets-controls/README.md @@ -4,7 +4,7 @@ Step 4 first-run input matrix, Step 5 snapshot/export, Step 7 provider-header and PR #95 B2 redaction controls on physical source copies. `--mechanism ` supports diagnosis only; selected runs retain `complete: false`. The current manifest -declares **66 independent controls and 416 required named assertion witnesses**. +declares **72 independent controls and 430 required named assertion witnesses**. These counts describe the workload; a declaration is not evidence that its removal passed. Actual fixed/removal/restored results belong to the coordinator's summary. @@ -177,8 +177,8 @@ modern `sk-` body characters, joined-span application, the original-text left-bo check, split scheme labels, split API-header labels and raw `?`/`#` password recognition independently. Four LLM controls remove the before-masking LF/TAB probe, its original left-boundary check, split API-header recognition and raw-password recognition. -Every core phase executes the complete 150-case bounding file; every new LLM phase -executes the complete 174-case scrubber file. Independent ordinary tooling tests pin +Every core phase executes the complete 155-case bounding file; every new LLM phase +executes the complete 185-case scrubber file. Independent ordinary tooling tests pin these inventories, the 278-case overlap cohort, 180-case CLI diagnostic cohort and 18-case command-export cohort. Actual collection still has to match in every phase. @@ -259,14 +259,14 @@ previous-word prefilter and the rule's own boundary predicate; retaining either would leave the tested policy active. Header/key recovery removals still change their original-rule boundary only, and all removals retain the same canonical matchers and bounded raw/joined standalone visitors. Current collection is 25 -complete files / 1,414 cases, with each control's exact workload and named target +complete files / 1,431 cases, with each control's exact workload and named target checked separately. These are updated declarations and author checks, not new physical causal captures or a clean review round. R3 is preserved as NONCLEAN; later corrective acceptance belongs to the owning phase register. Prior selected `7e9096a8` captures retain their older source/workload attribution. -The current 66-control / 198-phase full declaration projects 1,195 retained files -(66 × 18 + 7) under the unchanged 2,000-file / 64-MiB export bounds. This is a capacity +The current 72-control / 216-phase full declaration projects 1,303 retained files +(72 × 18 + 7) under the unchanged 2,000-file / 64-MiB export bounds. This is a capacity calculation, not a measured full capture. Historical 41-control captures retain their original source and cohort attribution; selected B2 or original-boundary recovery captures are not full-family acceptance. @@ -284,6 +284,14 @@ with its fixed and restored phases green and every declared witness failing on i that is author evidence, not a full-family capture. The eight-MiB scrubber cases now carry a 15-second case deadline instead of a wall-clock assertion, inside the phases' `CAUSAL_TIMEOUT_BUDGET`. +The increment's first review round added six more controls and 14 named assertions. Two +header controls narrow the label prefix back to one label, or its whitespace back to one +character, so a doubled `Bearer Bearer` label or a multi-character gap leaks the token. Two +controls per masker remove the host-shaped prefix's length bound (an admitted-size dotted +prefix then exhausts V8's RegExp stack, and its witness catches the throw) or its `^`/`$` +anchors (look-alike prefixes such as `john.doe:s3cret` turn public). The snapshot detector's +complete workload is now 11 cases. + The thin entry point uses the same coordinator, outcome and bounded process join as the filesystem family in [causal-controls](../causal-controls/README.md). The provider-header diagnostic control pins the store's fixed throwable message diff --git a/tools/secrets-controls/mutations.mjs b/tools/secrets-controls/mutations.mjs index 236ae471..3efdcf82 100644 --- a/tools/secrets-controls/mutations.mjs +++ b/tools/secrets-controls/mutations.mjs @@ -94,7 +94,21 @@ const headerSchemeLabelCases = [ ), 'masks every LF/TAB split position inside a scheme label after a header', 'masks the next token, failing closed, when a scheme label ends the line', + 'masks the token after stacked or quoted scheme labels in a header value', + 'masks the token after more than one whitespace character following a scheme label', ]; +const headerLabelValue = String.raw`(?:(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)\s+){0,8}["']?[^\s"',;}]+/gi,`; +const headerLine = (value) => + String.raw` /(^|[^A-Za-z0-9_-])((?:x-)?api-key|x-goog-api-key)(["']?\s*:\s*["']?)` + value; +const hostLengthGuard = ' if (query > HOST_SHAPED_MAX_LENGTH) return false;'; +const hostShapedLine = String.raw` /^(?:[a-z\d-]+(?:\.[a-z\d-]+)+\.?|localhost|\[[\da-f:.]+\])(?::\d{1,5})?$/i;`; +const hostShapedUnanchored = String.raw` /(?:[a-z\d-]+(?:\.[a-z\d-]+)+\.?|localhost|\[[\da-f:.]+\])(?::\d{1,5})?/i;`; +const hostLookalikeCases = [ + 'https://john.doe:s3cret?x=1@h.example/', + 'https://localhostuser:pw?a=b@h.example/', + 'https://user:pw.evil.com?x=1@h.example/', + 'https://h.example:123456?x=1@h.example/', +].map((input) => `masks userinfo whose prefix only resembles a host: ${input}`); const userinfoContactMarker = 'SECRETS_B2_USERINFO_CONTACT'; const coreUserinfoContactMarker = 'SECRETS_B2_CORE_USERINFO_CONTACT'; const contactAssignmentCases = (secondInput) => @@ -276,7 +290,7 @@ function coreProjectionControl(id, before, after, targets, marker = coreProjecti before, after, coreProjectionCases, - 150, + 155, targets[0], marker, targets.slice(1).map((target) => ({ target, marker })), @@ -289,7 +303,7 @@ function diagnosticControl(id, before, after, targets, marker = diagnosticMarker before, after, ['packages/llm/src/scrub-secrets.test.ts'], - 174, + 185, targets[0], marker, targets.slice(1).map((target) => ({ target, marker })), @@ -805,7 +819,7 @@ export const mutations = [ ' if (scrubSecrets(probe) !== probe) matches += 1;', ' void probe;', ['apps/cli/src/authoring/snapshot-notice.test.ts'], - 10, + 11, 'detects existing credential shapes without values or arbitrary keys in the notice', 'SNAPSHOT_CAUSAL_DETECTOR', ), @@ -1029,11 +1043,55 @@ export const mutations = [ // PR95: a header value's leading Bearer/Basic label is masked with its token. diagnosticControl( 'diagnostic-header-scheme-label', - String.raw` /(^|[^A-Za-z0-9_-])((?:x-)?api-key|x-goog-api-key)(["']?\s*:\s*["']?)(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r\s+|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c\s+)?[^\s"',;}]+/gi,`, - String.raw` /(^|[^A-Za-z0-9_-])((?:x-)?api-key|x-goog-api-key)(["']?\s*:\s*["']?)[^\s"',;}]+/gi,`, + headerLine(headerLabelValue), + headerLine(String.raw`[^\s"',;}]+/gi,`), headerSchemeLabelCases, headerSchemeLabelMarker, ), + // PR95 round 1: stacked labels (`Bearer Bearer `) and multi-character label whitespace. + diagnosticControl( + 'diagnostic-header-stacked-labels', + headerLine(headerLabelValue), + headerLine(headerLabelValue.replace(String.raw`\s+){0,8}`, String.raw`\s+)?`)), + ['masks the token after stacked or quoted scheme labels in a header value'], + headerSchemeLabelMarker, + ), + diagnosticControl( + 'diagnostic-header-label-whitespace', + headerLine(headerLabelValue), + headerLine(headerLabelValue.replace(String.raw`\s+){0,8}`, String.raw`\s){0,8}`)), + ['masks the token after more than one whitespace character following a scheme label'], + headerSchemeLabelMarker, + ), + // PR95 round 1: the host-shaped prefix is bounded before the RegExp and anchored at both ends. + diagnosticControl( + 'diagnostic-userinfo-host-length', + hostLengthGuard, + '', + ['masks an admitted-size dotted userinfo prefix without exhausting the RegExp stack'], + 'SECRETS_B2_USERINFO_HOST_LENGTH', + ), + coreProjectionControl( + 'core-userinfo-host-length', + hostLengthGuard, + '', + ['masks an admitted-size dotted userinfo prefix without exhausting the RegExp stack'], + 'SECRETS_B2_CORE_USERINFO_HOST_LENGTH', + ), + diagnosticControl( + 'diagnostic-userinfo-host-anchors', + hostShapedLine, + hostShapedUnanchored, + hostLookalikeCases, + userinfoContactMarker, + ), + coreProjectionControl( + 'core-userinfo-host-anchors', + hostShapedLine, + hostShapedUnanchored, + hostLookalikeCases, + coreUserinfoContactMarker, + ), // PR95: raw ?/# userinfo is public only after a host-shaped prefix AND an assignment. diagnosticControl( 'diagnostic-userinfo-contact-assignment', @@ -1048,7 +1106,7 @@ export const mutations = [ contactAssignment, String.raw` return userinfo.includes('=', query + 1);`, ['packages/llm/src/scrub-secrets.test.ts'], - 174, + 185, contactHostCase, userinfoContactMarker, rawPasswordAssignments.map((target) => ({ target, marker: diagnosticMarker })), @@ -1066,7 +1124,7 @@ export const mutations = [ contactAssignment, String.raw` return userinfo.includes('=', query + 1);`, coreProjectionCases, - 150, + 155, contactHostCase, coreUserinfoContactMarker, rawPasswordAssignments.map((target) => ({ target, marker: coreProjectionMarker })), From bdd292a6e5c92a18e161a3868deb602d083f0fc0 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Tue, 20 Oct 2026 02:41:45 +0300 Subject: [PATCH 11/97] fix(llm): address the second PR95 review round The second review round found that quote characters beside a Bearer/Basic label let the api-key header rule take the label as its token and leak the real one, that the eight-label bound failed open past its limit, and that several bounds and documented behaviours were unpinned or misdescribed. - The header rule accepts (escaped) quotes before every label and before the token, and its token is never another label or an already-masked [REDACTED]. A ninth label therefore fails the rule instead of leaking the token; the label rules and the whitespace-stripped residual probe then blank the whole diagnostic. - The standalone Bearer/Basic rules no longer stack labels: a label never takes another label (or a masked marker) as its token, so any number of stacked labels leaves the last one to mask the token, with no repetition to bound. Masked header, Bearer and Basic tokens are fixed points, so a templated `x-api-key: Bearer {{secrets.key}}` stays quiet in the authored-content notice. - Tests pin the eight-label bound and its fail-closed side, admitted-size label runs, the 260-character host bound (260 kept, 261 masked) and case-insensitivity in both maskers, and core's documented authority-delimiter difference from the LLM backstop. - Seven new secrets causal controls (79 controls, 441 witnesses), with updated provenance pins and README counts. - error-handling.md, built-in-tools.md and the bounding.ts docblock describe the label shapes, fixed points and core's authority delimiters exactly. The phase register records ruling C2's production-key constraint, the #308 DONE marker and the split-row IDs; deferred-tasks.md schedules the key constraint for Wave 4b. Refs: ADR-0113 Co-Authored-By: Claude Opus 5.5 --- .../cli/src/authoring/snapshot-notice.test.ts | 3 +- docs/reference/shared-core/built-in-tools.md | 8 +- docs/roadmap/deferred-tasks.md | 8 ++ .../phase-2.5.5-hardening-and-remediation.md | 10 +- docs/standards/error-handling.md | 23 +++-- packages/core/src/tools/bounding.test.ts | 24 ++++- packages/core/src/tools/bounding.ts | 7 +- packages/llm/src/scrub-secrets.test.ts | 99 ++++++++++++++++++- packages/llm/src/scrub-secrets.ts | 25 ++--- tools/causal-controls/provenance.test.mjs | 15 ++- tools/secrets-controls/README.md | 22 +++-- tools/secrets-controls/mutations.mjs | 91 +++++++++++++++-- 12 files changed, 276 insertions(+), 59 deletions(-) diff --git a/apps/cli/src/authoring/snapshot-notice.test.ts b/apps/cli/src/authoring/snapshot-notice.test.ts index df5e6393..78697f7c 100644 --- a/apps/cli/src/authoring/snapshot-notice.test.ts +++ b/apps/cli/src/authoring/snapshot-notice.test.ts @@ -48,13 +48,14 @@ describe('authored snapshot credential notice', () => { expect(err()).toBe(''); }); - it('keeps placeholder connection-string passwords quiet and reports a literal http(s) username', () => { + it('keeps placeholder passwords and header tokens quiet and reports a literal http(s) username', () => { const quiet = captureIo(); warnSnapshotLiterals( { env: { DATABASE_URL: 'postgres://app:{{secrets.db_password}}@db.example/app', CACHE_URL: 'redis://:{{secrets.cache_password}}@cache.example:6379', + AUTH_HEADER: 'x-api-key: Bearer {{secrets.provider_key}}', }, }, quiet.io, diff --git a/docs/reference/shared-core/built-in-tools.md b/docs/reference/shared-core/built-in-tools.md index 23e31eae..7c29305c 100644 --- a/docs/reference/shared-core/built-in-tools.md +++ b/docs/reference/shared-core/built-in-tools.md @@ -98,10 +98,10 @@ the text before it is host-shaped (a dotted name or IPv4, `localhost` or a brack with an optional port) **and** an `=` assignment follows it; otherwise the userinfo is masked. This is the same predicate as the [LLM diagnostic backstop](../../standards/error-handling.md#diagnostic-secret-backstop), including its 260-character bound on the host-shaped prefix. The projection otherwise differs from -that backstop by design: its authority also ends at `"`, `'`, `,`, `}` and `]`; it recognizes any -scheme of up to 32 characters rather than a finite list; a userinfo with no password is kept as a -vanity handle below 16 characters and masked from 16 on, under every scheme; and it does not treat -an already-masked password as masked. +that backstop by design: its authority also ends at `"`, `'`, `,`, `}` and `]`, so userinfo that +contains one of them — an already-masked `[REDACTED]` password among them — is left as it is; it +recognizes any scheme of up to 32 characters rather than a finite list; and a userinfo with no +password is kept as a vanity handle below 16 characters and masked from 16 on, under every scheme. LF/TAB inside a recognized token-shaped credential extends the masked original span. An ambiguous token-alphabet continuation after LF/TAB is conservatively masked with diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index f97a4290..373ecb20 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -2251,6 +2251,14 @@ future test cannot silently re-acquire it. ## Wave 3 approved later-wave ownership — 2026-10-16 +- [ ] **Scheduled → Wave 4b: production recipient key constraints.** Approved with the Wave 3 + Steps 9–19 rulings (2026-10-19, [ruling C2](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19)): + the confirmed production recipient for encrypted release custody must be an OpenPGP v4 key + with a cv25519 or RSA≥3072 encryption-capable subkey, and the release runner records its + GnuPG version (≥ 2.2.8) in the binding receipt. Wave 3 Step 9 implements custody with + synthetic keys only; an actual release fails closed until this recipient, its offline + recovery check and backup ownership exist. + - [ ] **Scheduled → Wave 5a: alt-screen word wrapping (#61).** The maintainer approved retaining character-wrap replacement with render-v2/D18 and picker positioning #60, so that renderer restructuring and the palette sweep occur diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 38fc5160..f909759b 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -434,7 +434,7 @@ This substream is the development-process substrate itself — `.github/workflow - **Cap `InputHistory.entries`.** `input-history.ts:23`'s `entries` array has no upper bound, unlike the `MAX_TOKEN_CHARS`/`MAX_TOOL_LINES` siblings in the same directory. Add the same kind of bound. *(S · `apps/cli/src/render/tui/input-history.ts` · G4)* - **Cache per-cluster widths and debounce resize wrap-cache invalidation.** `viewport.ts:56`'s `graphemeWidth` calls the full `string-width` library per already-segmented cluster, 2.5–6x slower than necessary on CJK/wide-script lines (`#64`); `chat-projection.ts:273`'s `entryWrapCache` fully invalidates on every `SIGWINCH` tick during a drag-resize instead of only the final size, compounding the cost (`#66`). Cache per-cluster widths and debounce resize invalidation. *(M · `apps/cli/src/render/tui/viewport.ts` · #64, #66)* - **Mount the four overlay components under adversarial render tests.** `PromptEditor`/`PaletteView`/`ReverseSearchView` (`prompt-view.tsx:1`, `G41`) and `MentionView`/`EffortTierList` (`palette-view.tsx:1`, `G53`, overlapping heavily with G41's coverage) all carry explicit sanitization-safety claims in comments but are never mounted by any test in the repo. Add `ink-testing-library` render tests per component in one PR. *(M · `apps/cli/src/render/tui/prompt-view.tsx` · G41, G53)* -- **Make chat's own `git_commit` denial message satisfiable.** `packages/core/src/tools/registry.ts:292` emits a workflow-authored `human_gate` denial message into chat that asks the user to do something chat structurally cannot produce. Give chat its own denial copy. *(S · `packages/core/src/tools/registry.ts` · #308)* +- ✅ **DONE ([Wave 3 Step 5](#wave-3-step-5-acceptance--2026-10-16)).** **Make chat's own `git_commit` denial message satisfiable.** `packages/core/src/tools/registry.ts:292` emits a workflow-authored `human_gate` denial message into chat that asks the user to do something chat structurally cannot produce. Give chat its own denial copy. *(S · `packages/core/src/tools/registry.ts` · #308)* - **Align `chat-resume`/`chat-list`/`chat-export` with the CLI's noun-verb convention.** These three flat subcommands (`apps/cli/src/commands/specs.ts:131`) break the `gate list`/`provider add` pattern used everywhere else. Add `chat resume`/`chat list`/`chat export` as the canonical forms, keeping the flat spellings as deprecated aliases. *(M · `apps/cli/src/commands/specs.ts` · #309)* - **Add a persistent keyless-state indicator to the Home strip.** `apps/cli/src/home/home-store.ts:73`'s `HomeSnapshot` carries no field for "no provider key configured," so a skipped or cancelled onboarding wizard leaves no durable nudge once the wizard screen is gone. Add a keyless-state field and a persistent nudge row. *(S · `apps/cli/src/home/home-store.ts` · #51)* - **Make onboarding's model-binding failure recoverable and regression-tested.** A `writeDefaultModel` failure inside the wizard (`apps/cli/src/onboarding/wizard.ts:241`) silently re-introduces the exact provider/default-model mismatch the wizard exists to prevent (`#49`), and there is no end-to-end test proving the chosen provider actually binds the first chat's model (`apps/cli/src/home/drive-home.test.ts:562`, `#52`). Make the write retried/recoverable and add the regression test in the same change. *(M · `apps/cli/src/onboarding/wizard.ts` · #49, #52)* @@ -3704,14 +3704,14 @@ No credential values or request captures belong in tracked evidence. | 9d | **Recorded split — workflow, runbook and whole-Step-9 landing:** least-privilege actual workflow dependency graph, required artifact/platform aggregation and exact-artifact consumers, lost-staging refusal and the release runbook update. The whole-Step-9 cumulative review retains all 9a–9d changes with preservation traces into Step 8, and the dedicated archive/crypto security sitting closes the step. | | 10 | **CLI behaviour/discovery integration:** #45/#264/#265/#268/#13/#17/#18/#310/#309/#19 under ADR-0110, consuming Steps 6–9. Production owner category matrix, real root/subtree JSON help, pricing/chat aliases, non-echoing Commander refusal and signal-safe deletion consent. G15 is refuted and receives real engine/store/log preservation evidence, with no invented token coalescing. Repeat affected packed gates. | | 10a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — diagnostic policy and output owners:** #45/#264/#265/#268 policy wiring, the production output-owner category registry with a drift test, verbose detail whose frames come from a same-build allowlist (no second build or post-finalization write), removal of the raw stack branch with G36/G46 re-scoped to its replacement, and the PR #95 lazy-dispatch cause-wrap and purity-root fold-ins. | -| 10b | **Recorded split — descriptor grammar and JSON help:** canonical chat paths and pricing spellings with working aliases and conflict refusal, non-echoing Commander refusal with the #311 static hint family (moved here from row 17; certified in 17a), excess-argument refusal, subcommand global help, real root/subtree JSON help (#13, including `help [path] --json`) and the purity guard. | +| 10b | **Recorded split — descriptor grammar and JSON help:** canonical chat paths and pricing spellings with working aliases and conflict refusal (#309/#18/#310), non-echoing Commander refusal with the #311 static hint family (moved here from row 17; certified in 17a), excess-argument refusal, subcommand global help (#17), real root/subtree JSON help (#13, including `help [path] --json`) and the purity guard. | | 10c | **Recorded split — consent, G15 evidence and landing:** signal-safe key-deletion consent (#19), G15 engine/store/log preservation evidence, promotion of the implemented Steps 6–10 commands.md sections (moved here from Step 19, per landing checklist item 3) and the repeated packed gates. | | 11 | **Chat normalization and abort:** #236/#237/#242/#307 → #218 → #21/#23. Canonical token/money source with approved additive LLM export, preserved two-decimal cap formatting and real immediate abort while a turn is blocked. Audit Home's imports. | | 11a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged, G54 moved here from row 17) — canonical token/money source:** #236/#237/#242/#307, including USD presentation of the conservative-hold disclosures, G54's character-based attachment estimate on the same helper, then #218 reseat vocabulary. | | 11b | **Recorded split — immediate abort:** #21/#23 certification and blocked-turn abort, including Ctrl-C mapped to `/cancel` while a tool approval is pending (an approved ADR-0057 amendment: the approval is rejected, never accepted), with an approval-path security sitting. | | 12 | **Chat decomposition:** #220/#244 after preceding chat fixes. Split lifecycle/drivers, line handler, printers and reseat/clear ownership. Preserve compatibility entry points, W7/W8 receipts, unknown-usage, full-screen bounds, disclosure activation and cancellation. | -| 12a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — chat leaves:** driver contracts, pure surface decisions, printers, headless drivers, the lifecycle seam and live notices move out of `commands/chat.ts` with no behaviour change. | -| 12b | **Recorded split — line handler and wiring:** #244 line-handler decomposition, session wiring, clear/reseat ownership, the REPL loop and command bodies. `commands/chat.ts` keeps only the command entry points and their argument/dependency types, importers are repointed, and `chat.ts`-internal duplication is consolidated (Home's chat wiring moves onto the shared helpers in 13c). | +| 12a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — chat leaves:** driver contracts, pure surface decisions, printers, headless drivers, the lifecycle seam and live notices move out of `commands/chat.ts` with no behaviour change (the first part of #220). | +| 12b | **Recorded split — line handler and wiring:** #244 line-handler decomposition and the rest of #220's session wiring, clear/reseat ownership, the REPL loop and command bodies. `commands/chat.ts` keeps only the command entry points and their argument/dependency types, importers are repointed, and `chat.ts`-internal duplication is consolidated (Home's chat wiring moves onto the shared helpers in 13c). | | 13 | **Chat domain policy:** G30/ADR-0112 after decomposition, all fresh/resume/Home/agent/rebuild paths. Invocation-live policy owner shared by old/pending hosts, post-approval/post-prepare target guard, pinned global selection, suspension/recovery and latest-only build publication. Actual failed/overlapping Home rebuild and approval-race controls, alongside no-authority/SSRF/ports; separate egress/config-trust security sitting. | | 13a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — pure substrate:** the DNS-hostname grammar and comparator (also rejecting `.internal`, which the SSRF floor always blocks), the strict `[chat_http].allowed_domains` key, the frozen dispatch target and the additive pre-dispatch check. | | 13b | **Recorded split — invocation authority:** the pinned global selection and the invocation-live, monotonically narrowing HTTP authority owner wired into standalone chat fresh/resume/clear/reseat and `agent run`. | @@ -9399,7 +9399,7 @@ plan amendment (row PR95) and the recorded splits; ID coverage is unchanged apar | Reassignments | G54 moves from row 17 to 11a; #311's implementation moves to 10b and its certification stays in 17a; G36/G46 are re-scoped in 10a to the branch that replaces the raw-stack branch and are certified in 17b; promotion of the Steps 6–10 commands.md sections moves from Step 19 to 10c. | | Interim release guard | 9a.1 adds a fail-closed check to the `release.yml` publish job that refuses any non-release candidate. Consequence, acknowledged: no npm publication, including a v0.1.x patch, until 9d and Wave 4b production custody exist. | | PR95 scrubber residuals | Fixed before 9a: the `api-key:Bearer`/`Basic` leak; raw `?`/`#` userinfo masked unless the text before it is host-shaped and an `=` assignment separates it from the authority `@`, in both the LLM leaf and core's twin predicate (core's other documented differences are recorded, not removed); an extended finite scheme list that masks colon-bearing userinfo for the new schemes; the scanner invariants, prefilter and witnesses; the load-sensitive 8 MiB timing case. LOW-1 and LOW-2 become approved deferrals with their recorded triggers. | -| Step 9 mechanics | The encrypted plaintext container is an in-house strict USTAR (regular files, repository-relative paths) recoverable with standard `gpg -d \| tar -x`; GnuPG ≥ 2.2.8 becomes a local `pnpm run ci` prerequisite and its controls fail rather than skip when it is absent; release-mode staging artifacts are retained 30 days, package-test artifacts 7; product-copy map negative controls run as `check:map-controls` locally and in a required uncached CI job; the private manifest keys final JS as `dist/.js`. | +| Step 9 mechanics | The encrypted plaintext container is an in-house strict USTAR (regular files, repository-relative paths) recoverable with standard `gpg -d \| tar -x`; GnuPG ≥ 2.2.8 becomes a local `pnpm run ci` prerequisite and its controls fail rather than skip when it is absent; release-mode staging artifacts are retained 30 days, package-test artifacts 7; product-copy map negative controls run as `check:map-controls` locally and in a required uncached CI job; the private manifest keys final JS as `dist/.js`; the production recipient must be an OpenPGP v4 key with a cv25519 or RSA≥3072 encryption subkey, a Wave 4b prerequisite recorded in [deferred-tasks.md](../deferred-tasks.md#wave-3-approved-later-wave-ownership--2026-10-16). | | Step 10 compatibility | Verbose frames use a same-build allowlist (a tsup `renderChunk` substitution of a fixed-length placeholder, so no second build or post-finalization write) and appear only on classified non-infrastructure diagnostics; `--quiet` treats live progress as decoration on TTY surfaces; excess positional arguments are refused on every non-variadic command; plain `note:` stderr lines become `{type:'diagnostic', code, message}` records in JSON mode; `help [path] --json` returns the discovery DTO; human `--help` keeps Commander behaviour for unknown grammar; deprecated aliases emit one structured JSON notice or one fixed human line kept under quiet and are hidden from human help listings. | | Steps 11–12 | The raw-µ¢ conservative-hold disclosures move to USD with their wording kept (a recorded #307 extension). Ctrl-C while a tool approval is pending maps to `/cancel`, rejecting the approval. `commands/chat.ts` keeps only its command entry points and their argument/dependency types after Step 12; `chat.ts`-internal duplication is consolidated in Step 12 and Home's chat wiring moves onto the shared helpers in 13c. | | Step 13 | `http_request` is omitted per turn while the live authority is empty or suspended; `[chat_http].allowed_domains` also rejects `.internal`. | diff --git a/docs/standards/error-handling.md b/docs/standards/error-handling.md index a4b314c0..93b5bbeb 100644 --- a/docs/standards/error-handling.md +++ b/docs/standards/error-handling.md @@ -130,10 +130,12 @@ content. - **Shapes.** OpenAI/Anthropic `sk-…` and Google `AIza…` key prefixes; a finite list of literal secret query names; the exact `api-key`, `x-api-key` and `x-goog-api-key` header names; and - `Bearer`/`Basic` authentication values. A header value that begins with up to eight - `Bearer`/`Basic` labels (a doubled `Bearer Bearer` is a common misconfiguration) is masked - together with the token that follows them, quoted or not; a standalone value accepts the same - stacked labels before its token. + `Bearer`/`Basic` authentication values. A header value that begins with up to eight (possibly + quoted, escaped or LF/TAB-split) `Bearer`/`Basic` labels — a doubled `Bearer Bearer` is a + common misconfiguration — is masked together with the token after them, and that token may + itself begin with (escaped) quotes, as in a JSON-encoded value. No label rule takes another + label as its token, so a standalone value with any number of stacked labels masks the token + after the last one, and an already-masked `[REDACTED]` token stays a fixed point. - **URL userinfo.** The `?`/`#` rule decides where userinfo ends: a raw `?` or `#` before the authority `@` begins public query or fragment text only when the text before it is host-shaped (a dotted name or IPv4, `localhost` or a bracketed IP literal, with an optional port, at most @@ -145,7 +147,8 @@ content. userinfo (`user:pass@`, `:pass@`), so a bare user such as `ssh://git@host` (or a colonless `redis://#TOKEN@host`) stays visible. Userinfo that is already masked stays as it is: an exactly masked password under a connection-string scheme, or every part masked or empty under - any scheme, so `postgres://app:{{secrets.password}}@db` projects to a fixed point. Core's + any scheme. The authored-content detector's projection of `postgres://app:{{secrets.password}}@db` + to `postgres://app:[REDACTED]@db` is therefore a fixed point. Core's journal projection applies the same `?`/`#` predicate ([built-in tools](../reference/shared-core/built-in-tools.md#finite-diagnostic-credential-masking)). - **Split and normalized forms.** Every rule except the literal query names also probes the @@ -158,13 +161,15 @@ content. place; an unlisted scheme ending in a listed one (`gftp://`, `jdbc:mysql://`) matches by its suffix; prose that the whitespace-stripped residual probe glues to a path-less URL and a later `@` blanks the whole diagnostic; a host fragment with no assignment (`https://host.example#sec@x`), - a single-label host query and an over-long host-shaped prefix are masked; and a - `Bearer`/`Basic` label that ends a line takes the next token with it. + a single-label host query and an over-long host-shaped prefix are masked; a + `Bearer`/`Basic` label that ends a line takes the next token with it; and nine or more labels + after a header name, or a token that begins with `,`, `;` or `}` after a label, blank the + whole diagnostic. - **Limits.** Userinfo under an unlisted scheme is not masked, including driver-qualified spellings such as `postgresql+psycopg2://`. Opaque `Bearer`/`Basic` values, `api-key` header values and literal secret query values end at any whitespace, LF and TAB included, so a - fragment after such a break stays visible, as it does after an ordinary space; a header token - that begins with `,`, `;` or `}` after a label is likewise outside the shape. Malformed URLs + fragment after such a break stays visible, as it does after an ordinary space; a label glued to + a quoted token without whitespace (`Bearer""`) is likewise outside the shape. Malformed URLs (`https:/u:p@h`), a raw `/` inside a password and invisible characters inside a scheme are outside the finite shapes. diff --git a/packages/core/src/tools/bounding.test.ts b/packages/core/src/tools/bounding.test.ts index d16764a2..d47e7084 100644 --- a/packages/core/src/tools/bounding.test.ts +++ b/packages/core/src/tools/bounding.test.ts @@ -1190,10 +1190,20 @@ describe('PR95 raw ?/# userinfo in the journal projection', () => { } }); - it('keeps its documented short colonless-handle difference from the LLM backstop', () => { + it('keeps its documented differences from the LLM backstop', () => { // Core retains a colonless userinfo under 16 characters; the LLM diagnostic backstop masks it. expect(redactSecretShapedText('http://#TOKEN@h.example')).toBe('http://#TOKEN@h.example'); expect(scrubSecrets('http://#TOKEN@h.example')).toBe('http://[REDACTED]@h.example'); + // Core's authority also ends at `"`, `'`, `,`, `}` and `]`, so it leaves userinfo containing + // one of them as it is: an already-masked password, a bracketed-host prefix, a raw comma. + for (const [input, masked] of [ + ['https://app:[REDACTED]@h.example/', 'https://[REDACTED]@h.example/'], + ['https://[::1]:8443?u:TOKENTOKEN@h.example/', 'https://[REDACTED]@h.example/'], + ['https://user:pa,ssword@host.example/', 'https://[REDACTED]@host.example/'], + ] as const) { + expect(redactSecretShapedText(input), input).toBe(input); + expect(scrubSecrets(input), input).toBe(masked); + } }); it('agrees with the LLM backstop on every raw ?/# decision outside those differences', () => { @@ -1230,6 +1240,18 @@ describe('PR95 raw ?/# userinfo in the journal projection', () => { ); }); + it('bounds the host-shaped prefix at exactly 260 characters, case-insensitively', () => { + const host = (length: number) => `${'a'.repeat(length - 2)}.b`; + const retained = `connect https://${host(260)}?contact=alice@example.invalid failed`; + expect(redactSecretShapedText(retained), 'SECRETS_B2_CORE_USERINFO_HOST_LENGTH').toBe(retained); + expect( + redactSecretShapedText(`https://${host(261)}?contact=alice@example.invalid`), + 'SECRETS_B2_CORE_USERINFO_HOST_LENGTH', + ).toBe('https://[redacted]@example.invalid'); + const upper = 'connect https://PROVIDER.INVALID:8443?contact=alice@example.invalid failed'; + expect(redactSecretShapedText(upper)).toBe(upper); + }); + it('masks an admitted-size dotted userinfo prefix without exhausting the RegExp stack', () => { const labels = 'a.'.repeat(Math.floor((8 * 1024 * 1024 - 256 - 64) / 2)); const input = `upstream said https://${labels}a?x=1@h failed`; diff --git a/packages/core/src/tools/bounding.ts b/packages/core/src/tools/bounding.ts index 13c7e779..33ede502 100644 --- a/packages/core/src/tools/bounding.ts +++ b/packages/core/src/tools/bounding.ts @@ -447,9 +447,10 @@ const HOST_SHAPED_MAX_LENGTH = 260; * `=` assignment follows it; anything else may be a malformed username or password. This is * the same predicate as the LLM diagnostic backstop (`packages/llm/src/scrub-secrets.ts`). * The projection differs from that backstop by design: its authority also ends at `"`, `'`, `,`, - * `}` and `]`; it recognizes any scheme of up to 32 characters rather than a finite list; a - * userinfo with no password is a vanity handle below 16 characters and masked from 16 on, under - * every scheme; and it does not treat an already-masked password as masked. + * `}` and `]`, so userinfo containing one of them (an already-masked `[REDACTED]` password among + * them) is left as it is; it recognizes any scheme of up to 32 characters rather than a finite list; + * and a userinfo with no password is a vanity handle below 16 characters and masked from 16 on, + * under every scheme. */ function isPublicQueryContact(userinfo: string, query: number): boolean { // A longer prefix cannot be a hostname, and an unbounded dotted run would exhaust V8's RegExp diff --git a/packages/llm/src/scrub-secrets.test.ts b/packages/llm/src/scrub-secrets.test.ts index 7732a55d..f8e73822 100644 --- a/packages/llm/src/scrub-secrets.test.ts +++ b/packages/llm/src/scrub-secrets.test.ts @@ -487,7 +487,8 @@ describe('PR95 header values that begin with a Bearer/Basic label', () => { headerSchemeLabel(scrubSecrets('api-key: Bearer\nrequest failed')).toBe( 'api-key: [REDACTED] failed', ); - expect(scrubSecrets('api-key: Bearer')).toBe('api-key: [REDACTED]'); + // A bare label is no token: it stays visible rather than become the masked value. + expect(scrubSecrets('api-key: Bearer')).toBe('api-key: Bearer'); }); it('keeps a token masked when the scheme label precedes the header name', () => { @@ -653,14 +654,15 @@ describe('PR95 review round 1: label stacking, host bounds and masked userinfo', }); it('masks the token after stacked labels in a standalone scheme value', () => { + // The last label masks the token: no label is taken as another label's token. for (const [input, expected] of [ [ `Authorization: Bearer Bearer ${token} rejected`, - 'Authorization: Bearer [REDACTED] rejected', + 'Authorization: Bearer Bearer [REDACTED] rejected', ], - [`Authorization: Basic Basic ${token}`, 'Authorization: Basic [REDACTED]'], - [`Authorization: Bearer Basic ${token}`, 'Authorization: Bearer [REDACTED]'], - [`Authorization: Bearer B\nearer ${token}`, 'Authorization: Bearer [REDACTED]'], + [`Authorization: Basic Basic ${token}`, 'Authorization: Basic Basic [REDACTED]'], + [`Authorization: Bearer Basic ${token}`, 'Authorization: Bearer Basic [REDACTED]'], + [`Authorization: Bearer B\nearer ${token}`, 'Authorization: Bearer Bearer [REDACTED]'], ] as const) { expect(scrubSecrets(input)).toBe(expected); } @@ -721,3 +723,90 @@ describe('PR95 review round 1: label stacking, host bounds and masked userinfo', ); }); }); + +describe('PR95 review round 2: quoted labels, unbounded stacking and fixed points', () => { + const token = ['abcdef', 'ghijkl', '123456'].join(''); + const stacked = (value: unknown) => expect(value, 'SECRETS_B2_STACKED_LABEL'); + + it('masks quoted and escaped tokens beside header scheme labels', () => { + for (const [input, expected] of [ + [ + `request headers: {'x-api-key': '"Bearer ${token}"'}`, + `request headers: {'x-api-key': '[REDACTED]"'}`, + ], + [`{"x-api-key":"Bearer \\"${token}\\""}`, '{"x-api-key":"[REDACTED]""}'], + [`{"x-goog-api-key":"Basic Basic \\"${token}\\""}`, '{"x-goog-api-key":"[REDACTED]""}'], + [`{"x-api-key":"\\\\\\"Bearer ${token}\\\\\\""}`, '{"x-api-key":"[REDACTED]""}'], + [`{"x-api-key":"\\"${token}\\""}`, '{"x-api-key":"[REDACTED]""}'], + // A delimiter-led token after a label: the whitespace-stripped residual probe blanks it all. + [`api-key: Bearer ,${token}`, '[REDACTED]'], + ] as const) { + const output = scrubSecrets(input); + expect(output, 'SECRETS_B2_HEADER_SCHEME_LABEL').toBe(expected); + expect(output.includes(token), 'SECRETS_B2_HEADER_SCHEME_LABEL').toBe(false); + } + }); + + it('masks a header token after up to eight stacked labels and fails closed beyond them', () => { + for (let count = 1; count <= 12; count += 1) { + const labels = Array.from({ length: count }, (_, index) => (index % 2 ? 'Basic' : 'Bearer')); + const output = scrubSecrets(`api-key: ${labels.join(' ')} ${token}`); + stacked(output).toBe(count <= 8 ? 'api-key: [REDACTED]' : '[REDACTED]'); + } + }); + + it('masks the token after any number of stacked standalone labels', () => { + for (let count = 1; count <= 12; count += 1) { + for (const label of ['Bearer', 'Basic']) { + const kept = `${label} `.repeat(count - 1); + stacked(scrubSecrets(`Authorization: ${kept}${label} ${token}`)).toBe( + `Authorization: ${kept}${label} [REDACTED]`, + ); + } + } + }); + + it( + 'keeps admitted-size label runs linear and fails closed', + () => { + const run = 'Bearer '.repeat(Math.floor((8 * 1024 * 1024 - 256 - 64) / 7)); + let header = ''; + let standalone = ''; + expect(() => { + header = scrubSecrets(`api-key: ${run}${token}`); + standalone = scrubSecrets(`Authorization: ${run}${token}`); + }, 'SECRETS_B2_HEADER_LABEL_BOUND').not.toThrow(); + expect(header === '[REDACTED]', 'SECRETS_B2_HEADER_LABEL_BOUND').toBe(true); + expect( + standalone.endsWith('Bearer [REDACTED]') && !standalone.includes(token), + 'SECRETS_B2_HEADER_LABEL_BOUND', + ).toBe(true); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); + + it('keeps already-masked scheme tokens as fixed points', () => { + for (const input of [ + 'x-api-key: Bearer [REDACTED]', + 'x-api-key: Basic [REDACTED]', + '{"x-api-key":"Bearer [REDACTED]"}', + '{"x-api-key":"Basic [REDACTED]"}', + "{'api-key': 'Bearer [REDACTED]'}", + 'Authorization: Bearer "[REDACTED]"', + ]) { + expect(scrubSecrets(input), 'SECRETS_B2_MASKED_FIXED_POINT').toBe(input); + } + }); + + it('bounds the host-shaped prefix at exactly 260 characters, case-insensitively', () => { + const host = (length: number) => `${'a'.repeat(length - 2)}.b`; + const retained = `connect https://${host(260)}?contact=alice@example.invalid failed`; + expect(scrubSecrets(retained), 'SECRETS_B2_USERINFO_HOST_LENGTH').toBe(retained); + expect( + scrubSecrets(`https://${host(261)}?contact=alice@example.invalid`), + 'SECRETS_B2_USERINFO_HOST_LENGTH', + ).toBe('https://[REDACTED]@example.invalid'); + const upper = 'connect https://PROVIDER.INVALID:8443?contact=alice@example.invalid failed'; + expect(scrubSecrets(upper)).toBe(upper); + }); +}); diff --git a/packages/llm/src/scrub-secrets.ts b/packages/llm/src/scrub-secrets.ts index dafa43b1..07460033 100644 --- a/packages/llm/src/scrub-secrets.ts +++ b/packages/llm/src/scrub-secrets.ts @@ -84,29 +84,32 @@ const NON_QUERY_SECRET_RULES: readonly SecretRule[] = [ anchor: { text: '://', window: URL_SCHEME_WINDOW }, }, // Exact literal header names, also in quoted diagnostic objects. No arbitrary header discovery. - // A value may begin with up to eight (possibly LF/TAB-split) Bearer/Basic labels, as a doubled - // `Bearer Bearer` misconfiguration does, and its token may be quoted: the labels and the token are - // masked together, so the later scheme-label rules cannot lose their anchor to this one. Like those - // rules, whitespace after a label may be a line break, and the next token is masked fail-closed. - // The repetition is bounded so that no admitted-size label run can exhaust the RegExp stack. + // A value may begin with up to eight (possibly quoted, escaped or LF/TAB-split) Bearer/Basic labels, + // as a doubled `Bearer Bearer` misconfiguration does: the labels and the token after them are masked + // together, and the token may itself begin with (escaped) quotes, as in a JSON-encoded value. The + // token is never another label, so a ninth label fails this rule rather than leak the token (the + // label rules and the residual probe then mask it), and never an already-masked `[REDACTED]`, which + // stays a fixed point. The repetition is bounded so no label run can exhaust the RegExp stack. { pattern: - /(^|[^A-Za-z0-9_-])((?:x-)?api-key|x-goog-api-key)(["']?\s*:\s*["']?)(?:(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)\s+){0,8}["']?[^\s"',;}]+/gi, + /(^|[^A-Za-z0-9_-])((?:x-)?api-key|x-goog-api-key)(["']?\s*:\s*["']?)(?:[\\"']*(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)\s+){0,8}(?![\\"']*(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)(?:\s|$))(?![\\"']*\[REDACTED\](?:[\s"',;}]|$))(?:\\*["'])?[^\s"',;}]+/gi, replacement: '$1$2$3[REDACTED]', joinedPattern: /(?:(?:x-)?api-key|x-goog-api-key)["']?\s*:/gi, boundary: /[A-Za-z0-9_-]/, }, // LF/TAB may split the finite scheme LABEL. Preserve the required label/value separator and - // ordinary following prose; opaque value fragments cannot be distinguished from that prose. Up to - // eight stacked labels (`Bearer Bearer `) precede the token without becoming it. + // ordinary following prose; opaque value fragments cannot be distinguished from that prose. A + // label never takes another label as its token, so any number of stacked labels + // (`Bearer Bearer `) leaves the last one to mask the token, with no repetition to bound, + // and an already-masked token stays a fixed point. { pattern: - /\bB[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r(?:\s+(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)){0,8}\s+\S+/gi, + /\bB[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r\s+(?![\\"']*(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)(?:\s|$))(?![\\"']*\[REDACTED\](?:[\s"',;}]|$))\S+/gi, replacement: 'Bearer [REDACTED]', }, { pattern: - /\bB[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c(?:\s+(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)){0,8}\s+\S+/gi, + /\bB[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c\s+(?![\\"']*(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)(?:\s|$))(?![\\"']*\[REDACTED\](?:[\s"',;}]|$))\S+/gi, replacement: 'Basic [REDACTED]', }, ]; @@ -168,7 +171,7 @@ function maskUrlUserinfo(match: string): string { if (colon < 0 && !colonless) return match; // Already-masked userinfo stays as it is: an exactly masked password under a connection-string // scheme, whose usernames are public, or a masked or empty username beside it under any scheme. - // `postgres://app:{{secrets.password}}@db` therefore projects to a fixed point. + // The authored-content detector's projection `postgres://app:[REDACTED]@db` is a fixed point. const username = userinfo.slice(0, Math.max(colon, 0)); if ( colon >= 0 && diff --git a/tools/causal-controls/provenance.test.mjs b/tools/causal-controls/provenance.test.mjs index 399d543f..e2ffd834 100644 --- a/tools/causal-controls/provenance.test.mjs +++ b/tools/causal-controls/provenance.test.mjs @@ -267,10 +267,10 @@ test('every declared filesystem and secrets anchor matches current production so }); test('secrets workload corrections retain independent complete-count and reader-witness pins', () => { - assert.equal(secretsControls.length, 72); + assert.equal(secretsControls.length, 79); assert.equal( secretsControls.reduce((count, control) => count + 1 + (control.witnesses ?? []).length, 0), - 430, + 441, ); for (const [id, file, expectedTests] of [ ['run-default-secret-reader', 'apps/cli/src/commands/run.test.ts', 84], @@ -313,7 +313,7 @@ test('secrets workload corrections retain independent complete-count and reader- 'core-userinfo-contact-host', 'core-userinfo-host-length', 'core-userinfo-host-anchors', - ].map((id) => [id, ['packages/core/src/tools/bounding.test.ts'], 155]), + ].map((id) => [id, ['packages/core/src/tools/bounding.test.ts'], 156]), ...[ 'diagnostic-joined-before-masking', 'diagnostic-joined-original-boundary', @@ -332,7 +332,14 @@ test('secrets workload corrections retain independent complete-count and reader- 'diagnostic-header-label-whitespace', 'diagnostic-userinfo-host-length', 'diagnostic-userinfo-host-anchors', - ].map((id) => [id, ['packages/llm/src/scrub-secrets.test.ts'], 185]), + 'diagnostic-header-label-bound', + 'diagnostic-header-token-not-label', + 'diagnostic-header-token-not-masked', + 'diagnostic-bearer-token-not-label', + 'diagnostic-basic-token-not-label', + 'diagnostic-bearer-token-not-masked', + 'diagnostic-basic-token-not-masked', + ].map((id) => [id, ['packages/llm/src/scrub-secrets.test.ts'], 191]), ...['diagnostic-inline-projection-order', 'diagnostic-original-control-boundary'].map((id) => [ id, ['apps/cli/src/render/sanitize.test.ts', 'apps/cli/src/process/render-error.test.ts'], diff --git a/tools/secrets-controls/README.md b/tools/secrets-controls/README.md index afe2f706..f36bab4e 100644 --- a/tools/secrets-controls/README.md +++ b/tools/secrets-controls/README.md @@ -4,7 +4,7 @@ Step 4 first-run input matrix, Step 5 snapshot/export, Step 7 provider-header and PR #95 B2 redaction controls on physical source copies. `--mechanism ` supports diagnosis only; selected runs retain `complete: false`. The current manifest -declares **72 independent controls and 430 required named assertion witnesses**. +declares **79 independent controls and 441 required named assertion witnesses**. These counts describe the workload; a declaration is not evidence that its removal passed. Actual fixed/removal/restored results belong to the coordinator's summary. @@ -128,7 +128,7 @@ permanent negative cases; no notice is produced before a successful export. Four snapshot controls remove the finite scrubber detection site and the actual `create`, `import`, and `run` warning calls. Each has a named value-free assertion in -its complete 10-case detector, 11-case create, 11-case import or 84-case run file. +its complete 11-case detector, 11-case create, 11-case import or 84-case run file. The adjacent tests check preserved authored files and admitted run snapshots, JSON stdout, and scan-budget/incomplete notices. These controls certify warning delivery for the planted supported shapes at these entry points, not universal secret @@ -177,8 +177,8 @@ modern `sk-` body characters, joined-span application, the original-text left-bo check, split scheme labels, split API-header labels and raw `?`/`#` password recognition independently. Four LLM controls remove the before-masking LF/TAB probe, its original left-boundary check, split API-header recognition and raw-password recognition. -Every core phase executes the complete 155-case bounding file; every new LLM phase -executes the complete 185-case scrubber file. Independent ordinary tooling tests pin +Every core phase executes the complete 156-case bounding file; every new LLM phase +executes the complete 191-case scrubber file. Independent ordinary tooling tests pin these inventories, the 278-case overlap cohort, 180-case CLI diagnostic cohort and 18-case command-export cohort. Actual collection still has to match in every phase. @@ -259,14 +259,14 @@ previous-word prefilter and the rule's own boundary predicate; retaining either would leave the tested policy active. Header/key recovery removals still change their original-rule boundary only, and all removals retain the same canonical matchers and bounded raw/joined standalone visitors. Current collection is 25 -complete files / 1,431 cases, with each control's exact workload and named target +complete files / 1,438 cases, with each control's exact workload and named target checked separately. These are updated declarations and author checks, not new physical causal captures or a clean review round. R3 is preserved as NONCLEAN; later corrective acceptance belongs to the owning phase register. Prior selected `7e9096a8` captures retain their older source/workload attribution. -The current 72-control / 216-phase full declaration projects 1,303 retained files -(72 × 18 + 7) under the unchanged 2,000-file / 64-MiB export bounds. This is a capacity +The current 79-control / 237-phase full declaration projects 1,429 retained files +(79 × 18 + 7) under the unchanged 2,000-file / 64-MiB export bounds. This is a capacity calculation, not a measured full capture. Historical 41-control captures retain their original source and cohort attribution; selected B2 or original-boundary recovery captures are not full-family acceptance. @@ -292,6 +292,14 @@ prefix then exhausts V8's RegExp stack, and its witness catches the throw) or it anchors (look-alike prefixes such as `john.doe:s3cret` turn public). The snapshot detector's complete workload is now 11 cases. +The second review round added seven controls and 11 named assertions. One unbounds the header +rule's label run (an admitted-size run of labels then exhausts the RegExp stack); two drop the +header token's guards, so a ninth label becomes the token and the real one leaks, or an +already-masked `Bearer [REDACTED]` stops being a fixed point; four drop the same two guards from +the standalone Bearer and Basic rules, so a stacked label becomes the token or a masked JSON +value loses its closing characters. The host-length controls also witness the exact 260/261 +boundary. + The thin entry point uses the same coordinator, outcome and bounded process join as the filesystem family in [causal-controls](../causal-controls/README.md). The provider-header diagnostic control pins the store's fixed throwable message diff --git a/tools/secrets-controls/mutations.mjs b/tools/secrets-controls/mutations.mjs index 3efdcf82..31f863eb 100644 --- a/tools/secrets-controls/mutations.mjs +++ b/tools/secrets-controls/mutations.mjs @@ -96,10 +96,22 @@ const headerSchemeLabelCases = [ 'masks the next token, failing closed, when a scheme label ends the line', 'masks the token after stacked or quoted scheme labels in a header value', 'masks the token after more than one whitespace character following a scheme label', + 'masks quoted and escaped tokens beside header scheme labels', ]; -const headerLabelValue = String.raw`(?:(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)\s+){0,8}["']?[^\s"',;}]+/gi,`; +const schemeLabel = String.raw`(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)`; +const notLabel = String.raw`(?![\\"']*${schemeLabel}(?:\s|$))`; +const notMasked = String.raw`(?![\\"']*\[REDACTED\](?:[\s"',;}]|$))`; +const headerLabelValue = String.raw`(?:[\\"']*${schemeLabel}\s+){0,8}${notLabel}${notMasked}(?:\\*["'])?[^\s"',;}]+/gi,`; const headerLine = (value) => String.raw` /(^|[^A-Za-z0-9_-])((?:x-)?api-key|x-goog-api-key)(["']?\s*:\s*["']?)` + value; +const labelLine = (first, value) => String.raw` /\b${first}\s+` + value; +const bearerFirst = String.raw`B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r`; +const basicFirst = String.raw`B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c`; +const labelToken = String.raw`${notLabel}${notMasked}\S+/gi,`; +const stackedLabelCase = + 'masks a header token after up to eight stacked labels and fails closed beyond them'; +const standaloneStackCase = 'masks the token after any number of stacked standalone labels'; +const fixedPointCase = 'keeps already-masked scheme tokens as fixed points'; const hostLengthGuard = ' if (query > HOST_SHAPED_MAX_LENGTH) return false;'; const hostShapedLine = String.raw` /^(?:[a-z\d-]+(?:\.[a-z\d-]+)+\.?|localhost|\[[\da-f:.]+\])(?::\d{1,5})?$/i;`; const hostShapedUnanchored = String.raw` /(?:[a-z\d-]+(?:\.[a-z\d-]+)+\.?|localhost|\[[\da-f:.]+\])(?::\d{1,5})?/i;`; @@ -290,7 +302,7 @@ function coreProjectionControl(id, before, after, targets, marker = coreProjecti before, after, coreProjectionCases, - 155, + 156, targets[0], marker, targets.slice(1).map((target) => ({ target, marker })), @@ -303,7 +315,7 @@ function diagnosticControl(id, before, after, targets, marker = diagnosticMarker before, after, ['packages/llm/src/scrub-secrets.test.ts'], - 185, + 191, targets[0], marker, targets.slice(1).map((target) => ({ target, marker })), @@ -1049,12 +1061,16 @@ export const mutations = [ headerSchemeLabelMarker, ), // PR95 round 1: stacked labels (`Bearer Bearer `) and multi-character label whitespace. - diagnosticControl( + control( 'diagnostic-header-stacked-labels', + scrubSecretsLeaf, headerLine(headerLabelValue), headerLine(headerLabelValue.replace(String.raw`\s+){0,8}`, String.raw`\s+)?`)), - ['masks the token after stacked or quoted scheme labels in a header value'], + ['packages/llm/src/scrub-secrets.test.ts'], + 191, + 'masks the token after stacked or quoted scheme labels in a header value', headerSchemeLabelMarker, + [{ target: stackedLabelCase, marker: 'SECRETS_B2_STACKED_LABEL' }], ), diagnosticControl( 'diagnostic-header-label-whitespace', @@ -1063,19 +1079,76 @@ export const mutations = [ ['masks the token after more than one whitespace character following a scheme label'], headerSchemeLabelMarker, ), + // PR95 round 2: the label run is bounded, its token is never another label or a masked span, + // and the standalone label rules take no label as their token. + diagnosticControl( + 'diagnostic-header-label-bound', + headerLine(headerLabelValue), + headerLine(headerLabelValue.replace(String.raw`\s+){0,8}`, String.raw`\s+)*`)), + ['keeps admitted-size label runs linear and fails closed'], + 'SECRETS_B2_HEADER_LABEL_BOUND', + ), + diagnosticControl( + 'diagnostic-header-token-not-label', + headerLine(headerLabelValue), + headerLine(headerLabelValue.replace(notLabel, '')), + [stackedLabelCase], + 'SECRETS_B2_STACKED_LABEL', + ), + diagnosticControl( + 'diagnostic-header-token-not-masked', + headerLine(headerLabelValue), + headerLine(headerLabelValue.replace(notMasked, '')), + [fixedPointCase], + 'SECRETS_B2_MASKED_FIXED_POINT', + ), + diagnosticControl( + 'diagnostic-bearer-token-not-label', + labelLine(bearerFirst, labelToken), + labelLine(bearerFirst, labelToken.replace(notLabel, '')), + [standaloneStackCase], + 'SECRETS_B2_STACKED_LABEL', + ), + diagnosticControl( + 'diagnostic-basic-token-not-label', + labelLine(basicFirst, labelToken), + labelLine(basicFirst, labelToken.replace(notLabel, '')), + [standaloneStackCase], + 'SECRETS_B2_STACKED_LABEL', + ), + diagnosticControl( + 'diagnostic-bearer-token-not-masked', + labelLine(bearerFirst, labelToken), + labelLine(bearerFirst, labelToken.replace(notMasked, '')), + [fixedPointCase], + 'SECRETS_B2_MASKED_FIXED_POINT', + ), + diagnosticControl( + 'diagnostic-basic-token-not-masked', + labelLine(basicFirst, labelToken), + labelLine(basicFirst, labelToken.replace(notMasked, '')), + [fixedPointCase], + 'SECRETS_B2_MASKED_FIXED_POINT', + ), // PR95 round 1: the host-shaped prefix is bounded before the RegExp and anchored at both ends. diagnosticControl( 'diagnostic-userinfo-host-length', hostLengthGuard, '', - ['masks an admitted-size dotted userinfo prefix without exhausting the RegExp stack'], + [ + 'masks an admitted-size dotted userinfo prefix without exhausting the RegExp stack', + 'bounds the host-shaped prefix at exactly 260 characters, case-insensitively', + ], 'SECRETS_B2_USERINFO_HOST_LENGTH', ), coreProjectionControl( 'core-userinfo-host-length', hostLengthGuard, '', - ['masks an admitted-size dotted userinfo prefix without exhausting the RegExp stack'], + [ + 'masks an admitted-size dotted userinfo prefix without exhausting the RegExp stack', + 'bounds the host-shaped prefix at exactly 260 characters, case-insensitively', + ], 'SECRETS_B2_CORE_USERINFO_HOST_LENGTH', ), diagnosticControl( @@ -1106,7 +1179,7 @@ export const mutations = [ contactAssignment, String.raw` return userinfo.includes('=', query + 1);`, ['packages/llm/src/scrub-secrets.test.ts'], - 185, + 191, contactHostCase, userinfoContactMarker, rawPasswordAssignments.map((target) => ({ target, marker: diagnosticMarker })), @@ -1124,7 +1197,7 @@ export const mutations = [ contactAssignment, String.raw` return userinfo.includes('=', query + 1);`, coreProjectionCases, - 155, + 156, contactHostCase, coreUserinfoContactMarker, rawPasswordAssignments.map((target) => ({ target, marker: coreProjectionMarker })), From 6fb7e9d9e9da934559b300adc2d01dd9f2e8d71e Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Tue, 20 Oct 2026 05:44:37 +0300 Subject: [PATCH 12/97] fix(llm): merge the masked ranges of the scrubber rules The third review round found that sequential rule application let one rule erase the context another needed: an api-key header token could swallow a Bearer/Basic label glued behind punctuation, and the round-2 masked-marker guard kept a tail glued to a mask (including the scrubber's own sk-/AIza masks) visible. A quote-closed label, quote runs, JSON-in-JSON headers and quoted or escaped placeholders were also handled incompletely, and a labels-only header value blanked the diagnostic depending on context. - Every non-query rule now finds its matches in the original text and the masked ranges of all rules are merged before anything is replaced, so no rule can erase another's context. A header or label rule masks only its named token group: header names, separators and labels stay visible, an LF/TAB-split label keeps its spelling, and masking an exact [REDACTED] is the identity, so the three masked-marker lookaheads are gone. - The header rule uses a zero-width boundary, accepts escaped quotes before its colon, skips up to eight punctuation-glued or quote-closed labels, refuses a run of up to eight concatenated labels or a nested header name as its token, and ends the token before a nested header name or at escapes before a delimiter. - The label rules accept a label closed by its own quote, refuse a glued or concatenated label as their token, and end the token before a label glued behind a non-word character. No token can start inside a quote run and every quantified label group is bounded, so admitted-size runs stay linear and stack-safe. - Tests pin each class, the quoted and escaped fixed points, the context-free labels-only value, core's ", ' and } delimiters and admitted-size escape and glued-label runs. 23 header and label controls replace ten (92 controls, 462 witnesses); provenance pins and README counts follow. - error-handling.md describes the merged masking, the shapes, the trade-offs and the " limit; commands.md notes quoted header placeholders; deferred-tasks.md and the rulings row record ruling C2 as approved. Refs: ADR-0113 Co-Authored-By: Claude Opus 5.5 --- .../cli/src/authoring/snapshot-notice.test.ts | 2 + docs/reference/cli/commands.md | 2 +- docs/roadmap/deferred-tasks.md | 3 +- .../phase-2.5.5-hardening-and-remediation.md | 2 +- docs/standards/error-handling.md | 38 ++- packages/core/src/tools/bounding.test.ts | 5 +- packages/llm/src/scrub-secrets.test.ts | 204 ++++++++++++--- packages/llm/src/scrub-secrets.ts | 149 ++++++++--- tools/causal-controls/provenance.test.mjs | 25 +- tools/secrets-controls/README.md | 32 ++- tools/secrets-controls/mutations.mjs | 233 ++++++++++++++---- 11 files changed, 544 insertions(+), 151 deletions(-) diff --git a/apps/cli/src/authoring/snapshot-notice.test.ts b/apps/cli/src/authoring/snapshot-notice.test.ts index 78697f7c..fbe6a0ac 100644 --- a/apps/cli/src/authoring/snapshot-notice.test.ts +++ b/apps/cli/src/authoring/snapshot-notice.test.ts @@ -56,6 +56,8 @@ describe('authored snapshot credential notice', () => { DATABASE_URL: 'postgres://app:{{secrets.db_password}}@db.example/app', CACHE_URL: 'redis://:{{secrets.cache_password}}@cache.example:6379', AUTH_HEADER: 'x-api-key: Bearer {{secrets.provider_key}}', + QUOTED_HEADER: 'x-api-key: Bearer "{{secrets.provider_key}}"', + QUOTED_BASIC: "x-api-key: Basic '{{secrets.provider_key}}'", }, }, quiet.io, diff --git a/docs/reference/cli/commands.md b/docs/reference/cli/commands.md index b4453f1c..16e8a757 100644 --- a/docs/reference/cli/commands.md +++ b/docs/reference/cli/commands.md @@ -234,7 +234,7 @@ Both are **surface-agnostic** git-native YAML operations — they read/write `.r The review visits at most 4,096 nodes through depth 64, checks strings of at most 16,384 UTF-16 code units and at most 262,144 checked code units in total, including property names. It reports only a count of strings matching the finite detector, the [diagnostic secret backstop](../../standards/error-handling.md#diagnostic-secret-backstop): prefixed OpenAI/Anthropic or Google keys, URL userinfo (any userinfo for HTTP(S)/WS(S)/FTP(S), colon-bearing userinfo for the listed connection-string schemes), exact `api-key`/`x-api-key`/`x-goog-api-key` headers, Bearer/Basic credentials and literal secret query names. It does not classify every credential format or distinguish arbitrary opaque credential fragments from following diagnostic prose. Over-budget strings or structure, repeated objects and unsupported object shapes produce a separate fixed incomplete-review notice. Object-key enumeration relies on the existing parser source ceiling; the node limit is not an independent bound on that enumeration. Neither notice includes values or arbitrary property names. -For this detector probe only, a recognised bare template reference without a path or filter is projected to an already masked span, provided its raw reference contains no recognised credential shape. This keeps `Bearer {{secrets.token}}`, equivalent query placeholders and a connection string whose password is a placeholder (`postgres://app:{{secrets.db_password}}@db`) quiet. An HTTP(S)/WS(S)/FTP(S) URL with a literal username beside a placeholder password still counts, because those schemes mask any userinfo. Adjacent literals, credential-shaped identifiers, paths, filter arguments and unknown or malformed references retain their evidence. This projection neither resolves references nor changes authored content. +For this detector probe only, a recognised bare template reference without a path or filter is projected to an already masked span, provided its raw reference contains no recognised credential shape. This keeps `Bearer {{secrets.token}}`, a header such as `x-api-key: Bearer "{{secrets.token}}"` (quoted or not), equivalent query placeholders and a connection string whose password is a placeholder (`postgres://app:{{secrets.db_password}}@db`) quiet. An HTTP(S)/WS(S)/FTP(S) URL with a literal username beside a placeholder password still counts, because those schemes mask any userinfo. Adjacent literals, credential-shaped identifiers, paths, filter arguments and unknown or malformed references retain their evidence. This projection neither resolves references nor changes authored content. ### `relavium logs ` diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index 373ecb20..c8ffad93 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -2254,8 +2254,7 @@ future test cannot silently re-acquire it. - [ ] **Scheduled → Wave 4b: production recipient key constraints.** Approved with the Wave 3 Steps 9–19 rulings (2026-10-19, [ruling C2](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19)): the confirmed production recipient for encrypted release custody must be an OpenPGP v4 key - with a cv25519 or RSA≥3072 encryption-capable subkey, and the release runner records its - GnuPG version (≥ 2.2.8) in the binding receipt. Wave 3 Step 9 implements custody with + with a cv25519 or RSA≥3072 encryption-capable subkey. Wave 3 Step 9 implements custody with synthetic keys only; an actual release fails closed until this recipient, its offline recovery check and backup ownership exist. diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index f909759b..b76a25d0 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -9399,7 +9399,7 @@ plan amendment (row PR95) and the recorded splits; ID coverage is unchanged apar | Reassignments | G54 moves from row 17 to 11a; #311's implementation moves to 10b and its certification stays in 17a; G36/G46 are re-scoped in 10a to the branch that replaces the raw-stack branch and are certified in 17b; promotion of the Steps 6–10 commands.md sections moves from Step 19 to 10c. | | Interim release guard | 9a.1 adds a fail-closed check to the `release.yml` publish job that refuses any non-release candidate. Consequence, acknowledged: no npm publication, including a v0.1.x patch, until 9d and Wave 4b production custody exist. | | PR95 scrubber residuals | Fixed before 9a: the `api-key:Bearer`/`Basic` leak; raw `?`/`#` userinfo masked unless the text before it is host-shaped and an `=` assignment separates it from the authority `@`, in both the LLM leaf and core's twin predicate (core's other documented differences are recorded, not removed); an extended finite scheme list that masks colon-bearing userinfo for the new schemes; the scanner invariants, prefilter and witnesses; the load-sensitive 8 MiB timing case. LOW-1 and LOW-2 become approved deferrals with their recorded triggers. | -| Step 9 mechanics | The encrypted plaintext container is an in-house strict USTAR (regular files, repository-relative paths) recoverable with standard `gpg -d \| tar -x`; GnuPG ≥ 2.2.8 becomes a local `pnpm run ci` prerequisite and its controls fail rather than skip when it is absent; release-mode staging artifacts are retained 30 days, package-test artifacts 7; product-copy map negative controls run as `check:map-controls` locally and in a required uncached CI job; the private manifest keys final JS as `dist/.js`; the production recipient must be an OpenPGP v4 key with a cv25519 or RSA≥3072 encryption subkey, a Wave 4b prerequisite recorded in [deferred-tasks.md](../deferred-tasks.md#wave-3-approved-later-wave-ownership--2026-10-16). | +| Step 9 mechanics | The encrypted plaintext container is an in-house strict USTAR (regular files, repository-relative paths) recoverable with standard `gpg -d \| tar -x`; GnuPG ≥ 2.2.8 becomes a prerequisite of local `pnpm run ci` and of the CI Ubuntu runner, documented in local-dev-setup.md, and its controls fail rather than skip when it is absent; release-mode staging artifacts are retained 30 days, package-test artifacts 7; product-copy map negative controls run as `check:map-controls` locally and in a required uncached CI job; the private manifest keys final JS as `dist/.js`; the production recipient must be an OpenPGP v4 key with a cv25519 or RSA≥3072 encryption subkey, a Wave 4b prerequisite recorded in [deferred-tasks.md](../deferred-tasks.md#wave-3-approved-later-wave-ownership--2026-10-16). | | Step 10 compatibility | Verbose frames use a same-build allowlist (a tsup `renderChunk` substitution of a fixed-length placeholder, so no second build or post-finalization write) and appear only on classified non-infrastructure diagnostics; `--quiet` treats live progress as decoration on TTY surfaces; excess positional arguments are refused on every non-variadic command; plain `note:` stderr lines become `{type:'diagnostic', code, message}` records in JSON mode; `help [path] --json` returns the discovery DTO; human `--help` keeps Commander behaviour for unknown grammar; deprecated aliases emit one structured JSON notice or one fixed human line kept under quiet and are hidden from human help listings. | | Steps 11–12 | The raw-µ¢ conservative-hold disclosures move to USD with their wording kept (a recorded #307 extension). Ctrl-C while a tool approval is pending maps to `/cancel`, rejecting the approval. `commands/chat.ts` keeps only its command entry points and their argument/dependency types after Step 12; `chat.ts`-internal duplication is consolidated in Step 12 and Home's chat wiring moves onto the shared helpers in 13c. | | Step 13 | `http_request` is omitted per turn while the live authority is empty or suspended; `[chat_http].allowed_domains` also rejects `.internal`. | diff --git a/docs/standards/error-handling.md b/docs/standards/error-handling.md index 93b5bbeb..eeb5efdb 100644 --- a/docs/standards/error-handling.md +++ b/docs/standards/error-handling.md @@ -129,13 +129,24 @@ message, and the backstop masks a finite set of secret **shapes**, not arbitrary content. - **Shapes.** OpenAI/Anthropic `sk-…` and Google `AIza…` key prefixes; a finite list of literal - secret query names; the exact `api-key`, `x-api-key` and `x-goog-api-key` header names; and - `Bearer`/`Basic` authentication values. A header value that begins with up to eight (possibly - quoted, escaped or LF/TAB-split) `Bearer`/`Basic` labels — a doubled `Bearer Bearer` is a - common misconfiguration — is masked together with the token after them, and that token may - itself begin with (escaped) quotes, as in a JSON-encoded value. No label rule takes another - label as its token, so a standalone value with any number of stacked labels masks the token - after the last one, and an already-masked `[REDACTED]` token stays a fixed point. + secret query names; the exact `api-key`, `x-api-key` and `x-goog-api-key` header names, also + in quoted, JSON-encoded and JSON-in-JSON (`\"x-api-key\":\"…\"`) objects; and + `Bearer`/`Basic` authentication values. A header value may begin with up to eight (possibly + quoted, escaped, punctuation-glued or LF/TAB-split) `Bearer`/`Basic` labels — a doubled + `Bearer Bearer` is a common misconfiguration. Its token may begin with any run of (escaped) + quotes and ends at a delimiter, at escapes before one, or before a nested header name, which + then masks its own value. A label may be closed by its own quote before its token + (`"Bearer" `). No label takes another label as its token, even one glued behind + punctuation, quoted or concatenated, so a standalone value with any number of stacked labels + masks the token after the last one, and a label glued to a token behind a non-word character + masks its own token. +- **Masking.** Every rule finds its matches in the original text, and the masked ranges of all + rules are merged before anything is replaced, so no rule can erase the context another rule + needs. A header or label rule masks only its token: the header name, the separator and any + labels stay visible, an LF/TAB-split label keeps its spelling, and quotes around a token stay + outside it. Masking a token that is already exactly `[REDACTED]` changes nothing, so a masked + token is a fixed point, quoted, escaped or after labels; text glued to the marker + (`[REDACTED];tail`) is masked with it. - **URL userinfo.** The `?`/`#` rule decides where userinfo ends: a raw `?` or `#` before the authority `@` begins public query or fragment text only when the text before it is host-shaped (a dotted name or IPv4, `localhost` or a bracketed IP literal, with an optional port, at most @@ -162,16 +173,19 @@ content. suffix; prose that the whitespace-stripped residual probe glues to a path-less URL and a later `@` blanks the whole diagnostic; a host fragment with no assignment (`https://host.example#sec@x`), a single-label host query and an over-long host-shaped prefix are masked; a - `Bearer`/`Basic` label that ends a line takes the next token with it; and nine or more labels - after a header name, or a token that begins with `,`, `;` or `}` after a label, blank the - whole diagnostic. + `Bearer`/`Basic` label that ends a line takes the next token with it; an opaque value after a + label ends only at whitespace, so in compact JSON it masks to the end of the run; and nine or + more labels after a header name, or a token that begins with `,`, `;` or `}` after a header + label, blank the whole diagnostic. Whether nine or more labels with no token after them blank + it can depend on the character before the header name, so a second scrub of such output may + blank what the first kept; no token is visible either way. - **Limits.** Userinfo under an unlisted scheme is not masked, including driver-qualified spellings such as `postgresql+psycopg2://`. Opaque `Bearer`/`Basic` values, `api-key` header values and literal secret query values end at any whitespace, LF and TAB included, so a fragment after such a break stays visible, as it does after an ordinary space; a label glued to a quoted token without whitespace (`Bearer""`) is likewise outside the shape. Malformed URLs - (`https:/u:p@h`), a raw `/` inside a password and invisible characters inside a scheme are - outside the finite shapes. + (`https:/u:p@h`), a raw `/` inside a password, invisible characters inside a scheme and JSON + whose quotes are themselves escaped as `\u0022` are outside the finite shapes. ## No silent catches diff --git a/packages/core/src/tools/bounding.test.ts b/packages/core/src/tools/bounding.test.ts index d47e7084..3a789333 100644 --- a/packages/core/src/tools/bounding.test.ts +++ b/packages/core/src/tools/bounding.test.ts @@ -1195,11 +1195,14 @@ describe('PR95 raw ?/# userinfo in the journal projection', () => { expect(redactSecretShapedText('http://#TOKEN@h.example')).toBe('http://#TOKEN@h.example'); expect(scrubSecrets('http://#TOKEN@h.example')).toBe('http://[REDACTED]@h.example'); // Core's authority also ends at `"`, `'`, `,`, `}` and `]`, so it leaves userinfo containing - // one of them as it is: an already-masked password, a bracketed-host prefix, a raw comma. + // one of them as it is: an already-masked password, a bracketed-host prefix, a raw delimiter. for (const [input, masked] of [ ['https://app:[REDACTED]@h.example/', 'https://[REDACTED]@h.example/'], ['https://[::1]:8443?u:TOKENTOKEN@h.example/', 'https://[REDACTED]@h.example/'], ['https://user:pa,ssword@host.example/', 'https://[REDACTED]@host.example/'], + ['https://user:pa"ssword@host.example/', 'https://[REDACTED]@host.example/'], + ["https://user:pa'ssword@host.example/", 'https://[REDACTED]@host.example/'], + ['https://user:pa}ssword@host.example/', 'https://[REDACTED]@host.example/'], ] as const) { expect(redactSecretShapedText(input), input).toBe(input); expect(scrubSecrets(input), input).toBe(masked); diff --git a/packages/llm/src/scrub-secrets.test.ts b/packages/llm/src/scrub-secrets.test.ts index f8e73822..371d9986 100644 --- a/packages/llm/src/scrub-secrets.test.ts +++ b/packages/llm/src/scrub-secrets.test.ts @@ -148,9 +148,12 @@ describe('finite diagnostic credential backstop regressions (PR #95 M2)', () => ...['\n', '\t'].flatMap((separator) => [ { input: `Bear${separator}er opaque-header-Z29 rejected`, - expected: 'Bearer [REDACTED] rejected', + expected: `Bear${separator}er [REDACTED] rejected`, + }, + { + input: `Bas${separator}ic dXNlcjpwYXNz rejected`, + expected: `Bas${separator}ic [REDACTED] rejected`, }, - { input: `Bas${separator}ic dXNlcjpwYXNz rejected`, expected: 'Basic [REDACTED] rejected' }, { input: `wss://ali${separator}ce:opaque@password@provider.invalid/v1`, expected: '[REDACTED]', @@ -202,8 +205,8 @@ describe('finite diagnostic credential backstop regressions (PR #95 M2)', () => for (const label of ['Bearer', 'Basic']) { for (const separator of ['\n', '\t', '\n\t']) { for (let position = 1; position < label.length; position += 1) { - const input = `${label.slice(0, position)}${separator}${label.slice(position)} opaque-header-Z29`; - expect(scrubSecrets(input)).toBe(`${label} [REDACTED]`); + const split = `${label.slice(0, position)}${separator}${label.slice(position)}`; + expect(scrubSecrets(`${split} opaque-header-Z29`)).toBe(`${split} [REDACTED]`); } } } @@ -456,12 +459,17 @@ describe('PR95 header values that begin with a Bearer/Basic label', () => { (name) => { for (const label of ['Bearer', 'Basic', 'bearer', 'BASIC']) { for (const [input, expected] of [ - [`${name}:${label} ${token}`, `${name}:[REDACTED]`], - [`header ${name}: ${label} ${token} rejected`, `header ${name}: [REDACTED] rejected`], - [`${name}:\t${label}\t${token}`, `${name}:\t[REDACTED]`], + [`${name}:${label} ${token}`, `${name}:${label} [REDACTED]`], + [ + `header ${name}: ${label} ${token} rejected`, + `header ${name}: ${label} [REDACTED] rejected`, + ], + [`${name}:\t${label}\t${token}`, `${name}:\t${label}\t[REDACTED]`], + // An opaque value after a label ends only at whitespace, so in compact JSON it fails closed + // to the end of the run, exactly as a standalone `"authorization":"Bearer …"` does. [ `{"${name}":"${label} ${token}","ordinary":"kept"}`, - `{"${name}":"[REDACTED]","ordinary":"kept"}`, + `{"${name}":"${label} [REDACTED]"}`, ], ] as const) { const output = scrubSecrets(input); @@ -476,8 +484,10 @@ describe('PR95 header values that begin with a Bearer/Basic label', () => { for (const label of ['Bearer', 'Basic']) { for (const separator of ['\n', '\t', '\n\t']) { for (let position = 1; position < label.length; position += 1) { - const input = `api-key:${label.slice(0, position)}${separator}${label.slice(position)} ${token}`; - headerSchemeLabel(scrubSecrets(input)).toBe('api-key:[REDACTED]'); + const split = `${label.slice(0, position)}${separator}${label.slice(position)}`; + headerSchemeLabel(scrubSecrets(`api-key:${split} ${token}`)).toBe( + `api-key:${split} [REDACTED]`, + ); } } } @@ -485,7 +495,7 @@ describe('PR95 header values that begin with a Bearer/Basic label', () => { it('masks the next token, failing closed, when a scheme label ends the line', () => { headerSchemeLabel(scrubSecrets('api-key: Bearer\nrequest failed')).toBe( - 'api-key: [REDACTED] failed', + 'api-key: Bearer\n[REDACTED] failed', ); // A bare label is no token: it stays visible rather than become the masked value. expect(scrubSecrets('api-key: Bearer')).toBe('api-key: Bearer'); @@ -627,13 +637,13 @@ describe('PR95 review round 1: label stacking, host bounds and masked userinfo', it('masks the token after stacked or quoted scheme labels in a header value', () => { for (const [input, expected] of [ - [`api-key: Bearer Bearer ${token}`, 'api-key: [REDACTED]'], - [`x-api-key: Basic Bearer ${token}`, 'x-api-key: [REDACTED]'], - [`{"x-api-key":"Bearer Bearer ${token}"}`, '{"x-api-key":"[REDACTED]"}'], - [`x-goog-api-key: Bearer\nBearer ${token}`, 'x-goog-api-key: [REDACTED]'], - [`api-key:Bearer B\tearer ${token}`, 'api-key:[REDACTED]'], - [`api-key: Bearer "${token}"`, 'api-key: [REDACTED]"'], - [`api-key: Basic '${token}'`, "api-key: [REDACTED]'"], + [`api-key: Bearer Bearer ${token}`, 'api-key: Bearer Bearer [REDACTED]'], + [`x-api-key: Basic Bearer ${token}`, 'x-api-key: Basic Bearer [REDACTED]'], + [`{"x-api-key":"Bearer Bearer ${token}"}`, '{"x-api-key":"Bearer Bearer [REDACTED]"}'], + [`x-goog-api-key: Bearer\nBearer ${token}`, 'x-goog-api-key: Bearer\nBearer [REDACTED]'], + [`api-key:Bearer B\tearer ${token}`, 'api-key:Bearer B\tearer [REDACTED]'], + [`api-key: Bearer "${token}"`, 'api-key: Bearer "[REDACTED]"'], + [`api-key: Basic '${token}'`, "api-key: Basic '[REDACTED]'"], ] as const) { const output = scrubSecrets(input); headerSchemeLabel(output).toBe(expected); @@ -643,9 +653,9 @@ describe('PR95 review round 1: label stacking, host bounds and masked userinfo', it('masks the token after more than one whitespace character following a scheme label', () => { for (const [input, expected] of [ - [`api-key: Bearer ${token} rejected`, 'api-key: [REDACTED] rejected'], - [`x-api-key:Basic \t${token}`, 'x-api-key:[REDACTED]'], - [`api-key:Bearer \r\n${token}`, 'api-key:[REDACTED]'], + [`api-key: Bearer ${token} rejected`, 'api-key: Bearer [REDACTED] rejected'], + [`x-api-key:Basic \t${token}`, 'x-api-key:Basic \t[REDACTED]'], + [`api-key:Bearer \r\n${token}`, 'api-key:Bearer \r\n[REDACTED]'], ] as const) { const output = scrubSecrets(input); headerSchemeLabel(output).toBe(expected); @@ -662,7 +672,7 @@ describe('PR95 review round 1: label stacking, host bounds and masked userinfo', ], [`Authorization: Basic Basic ${token}`, 'Authorization: Basic Basic [REDACTED]'], [`Authorization: Bearer Basic ${token}`, 'Authorization: Bearer Basic [REDACTED]'], - [`Authorization: Bearer B\nearer ${token}`, 'Authorization: Bearer Bearer [REDACTED]'], + [`Authorization: Bearer B\nearer ${token}`, 'Authorization: Bearer B\nearer [REDACTED]'], ] as const) { expect(scrubSecrets(input)).toBe(expected); } @@ -732,12 +742,18 @@ describe('PR95 review round 2: quoted labels, unbounded stacking and fixed point for (const [input, expected] of [ [ `request headers: {'x-api-key': '"Bearer ${token}"'}`, - `request headers: {'x-api-key': '[REDACTED]"'}`, + `request headers: {'x-api-key': '"Bearer [REDACTED]"'}`, + ], + [`{"x-api-key":"Bearer \\"${token}\\""}`, '{"x-api-key":"Bearer \\"[REDACTED]\\""}'], + [ + `{"x-goog-api-key":"Basic Basic \\"${token}\\""}`, + '{"x-goog-api-key":"Basic Basic \\"[REDACTED]\\""}', ], - [`{"x-api-key":"Bearer \\"${token}\\""}`, '{"x-api-key":"[REDACTED]""}'], - [`{"x-goog-api-key":"Basic Basic \\"${token}\\""}`, '{"x-goog-api-key":"[REDACTED]""}'], - [`{"x-api-key":"\\\\\\"Bearer ${token}\\\\\\""}`, '{"x-api-key":"[REDACTED]""}'], - [`{"x-api-key":"\\"${token}\\""}`, '{"x-api-key":"[REDACTED]""}'], + [ + `{"x-api-key":"\\\\\\"Bearer ${token}\\\\\\""}`, + '{"x-api-key":"\\\\\\"Bearer [REDACTED]\\\\\\""}', + ], + [`{"x-api-key":"\\"${token}\\""}`, '{"x-api-key":"\\"[REDACTED]\\""}'], // A delimiter-led token after a label: the whitespace-stripped residual probe blanks it all. [`api-key: Bearer ,${token}`, '[REDACTED]'], ] as const) { @@ -751,7 +767,7 @@ describe('PR95 review round 2: quoted labels, unbounded stacking and fixed point for (let count = 1; count <= 12; count += 1) { const labels = Array.from({ length: count }, (_, index) => (index % 2 ? 'Basic' : 'Bearer')); const output = scrubSecrets(`api-key: ${labels.join(' ')} ${token}`); - stacked(output).toBe(count <= 8 ? 'api-key: [REDACTED]' : '[REDACTED]'); + stacked(output).toBe(count <= 8 ? `api-key: ${labels.join(' ')} [REDACTED]` : '[REDACTED]'); } }); @@ -793,6 +809,12 @@ describe('PR95 review round 2: quoted labels, unbounded stacking and fixed point '{"x-api-key":"Basic [REDACTED]"}', "{'api-key': 'Bearer [REDACTED]'}", 'Authorization: Bearer "[REDACTED]"', + // Round 3: quoted and backslash-escaped masked tokens after a label or a header separator. + 'x-api-key: Bearer "[REDACTED]"', + "x-api-key: Basic '[REDACTED]'", + 'api-key: \\"[REDACTED]\\"', + 'api-key: Bearer \\"[REDACTED]\\"', + '{"x-api-key":"Bearer \\"[REDACTED]\\""}', ]) { expect(scrubSecrets(input), 'SECRETS_B2_MASKED_FIXED_POINT').toBe(input); } @@ -810,3 +832,129 @@ describe('PR95 review round 2: quoted labels, unbounded stacking and fixed point expect(scrubSecrets(upper)).toBe(upper); }); }); + +describe('PR95 review round 3: merged masking, glued labels and nested headers', () => { + const token = ['abcdef', 'ghijkl', '123456'].join(''); + const cases = (marker: string, pairs: readonly (readonly [string, string])[]) => { + for (const [input, expected] of pairs) { + const output = scrubSecrets(input); + expect(output, marker).toBe(expected); + expect(output.includes(token) || output.includes('SECRET'), marker).toBe(false); + expect(scrubSecrets(output), marker).toBe(output); + } + }; + + it("masks a tail glued to a masked token, including the scrubber's own masks", () => { + cases('SECRETS_B3_MASKED_TAIL', [ + [ + 'Authorization: Bearer sk-abcdefghijklmnopqrstuv;TENANTSECRET99', + 'Authorization: Bearer [REDACTED]', + ], + [ + 'Authorization: Basic AIzaSyABCDEFGHIJKLMNOPQRSTUVWX,TAILSECRET', + 'Authorization: Basic [REDACTED]', + ], + ["Authorization: Bearer [REDACTED]'TAILSECRET", 'Authorization: Bearer [REDACTED]'], + ['Authorization: Bearer [REDACTED]TAILSECRET', 'Authorization: Bearer [REDACTED]'], + [ + '{"authorization":"Bearer sk-proj-abcdefghijklmnopqrst","api_key":"SECRETVALUE77"}', + '{"authorization":"Bearer [REDACTED]"}', + ], + // A header token continues through a backslash, so a marker glued to it is no fixed point. + ['api-key: [REDACTED]\\TAILSECRET', 'api-key: [REDACTED]'], + ['api-key: sk-abcdefghijklmnopqrstuv\\TAILSECRET', 'api-key: [REDACTED]'], + ]); + }); + + it('masks the token after a label glued behind punctuation or inside another token', () => { + cases('SECRETS_B3_GLUED_LABEL', [ + [`api-key: \\"(Bearer ${token}`, 'api-key: \\"(Bearer [REDACTED]'], + [`x-api-key:""(Bearer ${token}`, 'x-api-key:""(Bearer [REDACTED]'], + [`api-key: \\"=Basic ${token}`, 'api-key: \\"=Basic [REDACTED]'], + [`api-key: \\"[REDACTED]Bearer ${token}`, 'api-key: \\"[REDACTED] [REDACTED]'], + [ + `Authorization: Bearer Bearer (Bearer ${token}`, + 'Authorization: Bearer Bearer (Bearer [REDACTED]', + ], + [`api-key: abc(Bearer ${token}`, 'api-key: [REDACTED] [REDACTED]'], + [`Bearer x"Bearer ${token}`, 'Bearer [REDACTED]"Bearer [REDACTED]'], + // A terminal control inside the run makes the raw and normalized views disagree: fail closed. + [`Bearer \u001b[0m""Bearer \n${token}`, '[REDACTED]'], + ]); + }); + + it('masks the token after a label closed by its own quote', () => { + cases('SECRETS_B3_QUOTED_LABEL', [ + [`x-api-key: "Bearer" ${token}`, 'x-api-key: "Bearer" [REDACTED]'], + [`api-key: 'Bearer' ${token}`, "api-key: 'Bearer' [REDACTED]"], + [`{"x-api-key": "Bearer" "${token}"}`, '{"x-api-key": "Bearer" "[REDACTED]"}'], + [`Authorization: Bearer "Bearer" ${token}`, 'Authorization: Bearer "Bearer" [REDACTED]'], + [`Authorization: Bearer "Basic" ${token}`, 'Authorization: Bearer "Basic" [REDACTED]'], + [ + `Authorization: Basic \\"Bearer\\" ${token}`, + 'Authorization: Basic \\"Bearer\\" [REDACTED]', + ], + ]); + // A word that only begins with a label is a token; a label at the end of the text has none. + expect(scrubSecrets('Authorization: Basic Basicx'), 'SECRETS_B3_QUOTED_LABEL').toBe( + 'Authorization: Basic [REDACTED]', + ); + expect(scrubSecrets('Authorization: Basic Basic'), 'SECRETS_B3_QUOTED_LABEL').toBe( + 'Authorization: Basic Basic', + ); + }); + + it('masks a header token after any run of quotes or escapes, also in JSON-encoded JSON', () => { + cases('SECRETS_B3_HEADER_ESCAPES', [ + [`x-api-key: '""${token}`, `x-api-key: '""[REDACTED]`], + [`{"x-api-key":"\\"\\"${token}"}`, '{"x-api-key":"\\"\\"[REDACTED]"}'], + [`{"x-api-key":"\\\\\\"${token}\\\\\\""}`, '{"x-api-key":"\\\\\\"[REDACTED]\\\\\\""}'], + [ + `{"body":"{\\"x-api-key\\":\\"${token}\\"}"}`, + '{"body":"{\\"x-api-key\\":\\"[REDACTED]\\"}"}', + ], + [ + `{"body":"{\\"x-api-key\\":\\"Bearer ${token}\\"}"}`, + '{"body":"{\\"x-api-key\\":\\"Bearer [REDACTED]\\"}"}', + ], + ]); + }); + + it('keeps every header value and label token apart from a nested header name', () => { + cases('SECRETS_B3_NESTED_HEADER', [ + [`api-key:""api-key":${token}`, 'api-key:""api-key":[REDACTED]'], + [`api-key: x(api-key: ${token}`, 'api-key: [REDACTED]api-key: [REDACTED]'], + [`api-key:(api-key:[REDACTED]${token}`, 'api-key:[REDACTED]api-key:[REDACTED]'], + [`Bearer api-key: ${token}`, 'Bearer [REDACTED] [REDACTED]'], + [`Bearer api-key:,${token}`, 'Bearer [REDACTED]'], + ]); + }); + + it('treats a labels-only header value alike in every context', () => { + for (const input of ['api-key: Bearer Bearer', 'x api-key: Bearer Bearer', 'api-key: Basic']) { + expect(scrubSecrets(input), 'SECRETS_B3_LABEL_CONTEXT').toBe(input); + } + const masked = scrubSecrets('sk-proj-ABCDEFGHIJKLMNOPQRSTUV api-key: Bearer Bearer'); + expect(masked, 'SECRETS_B3_LABEL_CONTEXT').toBe('[REDACTED] api-key: Bearer Bearer'); + expect(scrubSecrets(masked), 'SECRETS_B3_LABEL_CONTEXT').toBe(masked); + }); + + it( + 'keeps admitted-size escape and concatenated-label runs linear and total', + () => { + const size = 8 * 1024 * 1024 - 256 - 64; + const escapes = `api-key: ${'\\\\'.repeat(size)}`; + const quoted = `api-key: ${'"Bearer'.repeat(Math.floor(size / 7))} ${token}`; + const standalone = `Bearer ${'"Bearer'.repeat(Math.floor(size / 7))} ${token}`; + let results: string[] = []; + expect(() => { + results = [escapes, quoted, standalone].map((input) => scrubSecrets(input)); + }, 'SECRETS_B3_ADMITTED_RUNS').not.toThrow(); + // A run of escapes holds no token, and the token after either glued-label run is masked. + expect(results[0] === escapes, 'SECRETS_B3_ADMITTED_RUNS').toBe(true); + expect(results[1]?.includes(token), 'SECRETS_B3_ADMITTED_RUNS').toBe(false); + expect(results[2]?.includes(token), 'SECRETS_B3_ADMITTED_RUNS').toBe(false); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); +}); diff --git a/packages/llm/src/scrub-secrets.ts b/packages/llm/src/scrub-secrets.ts index 07460033..84f28cf2 100644 --- a/packages/llm/src/scrub-secrets.ts +++ b/packages/llm/src/scrub-secrets.ts @@ -13,8 +13,8 @@ import { splitProbe, type SplitProbe, type SplitProbeAnchor } from './split-cred * diagnostic. The finite patterns check boundaries in the original text (the URL rule has none, like the * pre-PR matcher, so a scheme glued to a word character is masked in place) and preserve already-masked spans; * this is not universal fragment discovery. Query names are excluded from the residual probe. - * Pure and total for string input: regex replacements and a - * finite residual check, with no parsing or external calls on the error path. + * Pure and total for string input: regex matching, one merged masking pass and a finite residual + * check, with no parsing or external calls on the error path. * Asserted by `llm-error.test.ts` (the backstop) and each adapter's * secret-safety test (a planted secret → a secret-free surfaced `LlmError`); see security-review.md. * The canonical policy is error-handling.md's diagnostic secret backstop. @@ -23,21 +23,41 @@ export function scrubSecrets(text: string): string { return scrubQuerySecrets(scrubNonQuerySecrets(text)); } +/** A range of the original diagnostic that one rule masks with `[REDACTED]`. */ +interface MaskRange { + readonly start: number; + readonly end: number; +} + interface SecretRule { readonly pattern: RegExp; - readonly replacement: string | ((match: string) => string); + /** The masked range of one match, relative to the match; `undefined` keeps the match. */ + readonly mask: (match: RegExpExecArray) => MaskRange | undefined; + /** Whether a joined-view candidate is a credential; absent means every candidate is. */ + readonly credential?: (matched: string) => boolean; readonly joinedPattern?: RegExp; readonly boundary?: RegExp; /** Split-scanner prefilter: a match must reach this literal within its window (see `splitProbe`). */ readonly anchor?: SplitProbeAnchor; } +/** Mask the whole match. */ +const WHOLE_MATCH = (match: RegExpExecArray): MaskRange => ({ start: 0, end: match[0].length }); + +/** Mask only the named `token` group of a `d`-flagged pattern, keeping its label or header around it. */ +function tokenGroup(match: RegExpExecArray): MaskRange | undefined { + const span = match.indices?.groups?.['token']; + return span === undefined + ? undefined + : { start: span[0] - match.index, end: span[1] - match.index }; +} + /** * Deliberately NOT word-bounded: the pre-PR http(s) matcher masked `xhttps://user:pass@host`, and a `\b` * here would let any scheme glued to a preceding word character through unmasked. PR #95 added * ws/wss/ftp/ftps under the same rule, and the PR95 increment added the connection-string schemes * (postgres(ql), mysql, mariadb, redis(s), mongodb(+srv), amqp(s), ssh, sftp, ldap(s), smtp(s)), whose - * colonless userinfo names a user and stays visible (see `maskUrlUserinfo`). The rule has no `\b` prefix, + * colonless userinfo names a user and stays visible (see `urlUserinfoRange`). The rule has no `\b` prefix, * so it supplies its own joined form (the split-credential scanner would otherwise strip two characters * that are not there) and its `boundary` blocks no start. Consequences, pinned by tests and fail-closed: an * unlisted longer scheme name ending in a listed one (`gftp://u@h`) matches by suffix, and the @@ -61,17 +81,23 @@ const URL_SCHEME_WINDOW = 32; /** A start boundary that never matches: no preceding character makes a start ineligible. */ const NO_BOUNDARY = /(?!)/; -/** One private authority per existing credential family, shared by probes and output masking. */ +/** + * One private authority per existing credential family, shared by probes and output masking. Every + * rule finds its matches in the original diagnostic and the masked ranges of all rules are merged + * before anything is replaced, so no rule can erase the context another rule needs, whatever their + * order. A token rule masks only its token, so its label or header name stays visible, and masking a + * token that is already exactly `[REDACTED]` changes nothing: every masked token is a fixed point. + */ const NON_QUERY_SECRET_RULES: readonly SecretRule[] = [ // OpenAI sk-/sk-proj-/sk-svcacct- and Anthropic sk-ant- key prefixes. { pattern: /\bsk-(?=[\w-]{16})[\w-]+/g, - replacement: '[REDACTED]', + mask: WHOLE_MATCH, boundary: /\w/, }, { pattern: /\bAIza(?=[\w-]{20})[\w-]+/g, - replacement: '[REDACTED]', + mask: WHOLE_MATCH, boundary: /\w/, }, // Finite authority schemes, not URL validation. Raw ?/# before the first @ may belong @@ -79,38 +105,43 @@ const NON_QUERY_SECRET_RULES: readonly SecretRule[] = [ { pattern: URL_USERINFO_PATTERN, joinedPattern: URL_USERINFO_PATTERN, - replacement: maskUrlUserinfo, + mask: (match) => urlUserinfoRange(match[0]), + credential: (matched) => urlUserinfoRange(matched) !== undefined, boundary: NO_BOUNDARY, anchor: { text: '://', window: URL_SCHEME_WINDOW }, }, - // Exact literal header names, also in quoted diagnostic objects. No arbitrary header discovery. - // A value may begin with up to eight (possibly quoted, escaped or LF/TAB-split) Bearer/Basic labels, - // as a doubled `Bearer Bearer` misconfiguration does: the labels and the token after them are masked - // together, and the token may itself begin with (escaped) quotes, as in a JSON-encoded value. The - // token is never another label, so a ninth label fails this rule rather than leak the token (the - // label rules and the residual probe then mask it), and never an already-masked `[REDACTED]`, which - // stays a fixed point. The repetition is bounded so no label run can exhaust the RegExp stack. + // Exact literal header names, also in quoted or JSON-escaped diagnostic objects. No arbitrary + // header discovery. A value may begin with up to eight Bearer/Basic labels (possibly quoted, + // escaped, glued behind punctuation or LF/TAB-split), as a doubled `Bearer Bearer` + // misconfiguration does; the token after them is masked and the labels stay. The token may begin + // with any run of (escaped) quotes, is never a label, a run of labels or another header name, and + // ends at a delimiter, at escapes before one or before a header name glued to it, which then masks + // its own value. The label repetition is bounded so no label run can exhaust the RegExp stack; + // past the bound the residual probe blanks the whole diagnostic. { pattern: - /(^|[^A-Za-z0-9_-])((?:x-)?api-key|x-goog-api-key)(["']?\s*:\s*["']?)(?:[\\"']*(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)\s+){0,8}(?![\\"']*(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)(?:\s|$))(?![\\"']*\[REDACTED\](?:[\s"',;}]|$))(?:\\*["'])?[^\s"',;}]+/gi, - replacement: '$1$2$3[REDACTED]', - joinedPattern: /(?:(?:x-)?api-key|x-goog-api-key)["']?\s*:/gi, + /(?[^\s"',;}\\][^\s"',;}]*?)(?=(?<=[^A-Za-z0-9_-])(?:(?:x-)?api-key|x-goog-api-key)\\*["']?\s*:|\\{0,15}[\s"',;}]|\\{0,15}$)/dgi, + mask: tokenGroup, + joinedPattern: /(?:(?:x-)?api-key|x-goog-api-key)\\*["']?\s*:/gi, boundary: /[A-Za-z0-9_-]/, }, - // LF/TAB may split the finite scheme LABEL. Preserve the required label/value separator and - // ordinary following prose; opaque value fragments cannot be distinguished from that prose. A - // label never takes another label as its token, so any number of stacked labels - // (`Bearer Bearer `) leaves the last one to mask the token, with no repetition to bound, - // and an already-masked token stays a fixed point. + // LF/TAB may split the finite scheme LABEL, and a label may be closed by its own quote before the + // whitespace (`"Bearer" `). Preserve the required label/value separator and ordinary + // following prose; opaque value fragments cannot be distinguished from that prose. A label never + // takes another label as its token, even one glued behind punctuation, quoted or concatenated, so + // any number of stacked labels leaves the last one to mask the token, and a token ends before a + // label glued to it behind a non-word character, which then masks its own token. Quotes and + // `,;}` around a token stay outside it unless it continues past them, so `[REDACTED];tail` is + // masked whole. { pattern: - /\bB[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r\s+(?![\\"']*(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)(?:\s|$))(?![\\"']*\[REDACTED\](?:[\s"',;}]|$))\S+/gi, - replacement: 'Bearer [REDACTED]', + /\bB[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r[\\"']*\s+(?![^\w\s]*(?:(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)[\\"']*){1,8}(?:\s|$))[\\"']*(?(?=[^\s\\"'])\S*?[^\s\\"',;}])[\\"',;}]*(?=(?<=[^\w])(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)[\\"']*(?:\s|$)|\s|$)/dgi, + mask: tokenGroup, }, { pattern: - /\bB[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c\s+(?![\\"']*(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)(?:\s|$))(?![\\"']*\[REDACTED\](?:[\s"',;}]|$))\S+/gi, - replacement: 'Basic [REDACTED]', + /\bB[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c[\\"']*\s+(?![^\w\s]*(?:(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)[\\"']*){1,8}(?:\s|$))[\\"']*(?(?=[^\s\\"'])\S*?[^\s\\"',;}])[\\"',;}]*(?=(?<=[^\w])(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)[\\"']*(?:\s|$)|\s|$)/dgi, + mask: tokenGroup, }, ]; @@ -144,7 +175,7 @@ const HOST_SHAPED_MAX_LENGTH = 260; * before it is host-shaped **and** an `=` assignment follows it, as in * `https://host.example?contact=a@b.example`. Otherwise the text is userinfo: `https://us?er:TOKEN@h/`, * `http://#TOKEN@h`, `https://my.user:1234?TOKEN@h/` and a host fragment without an assignment - * (`https://host.example#sec@x`, failing closed) are masked under the rest of `maskUrlUserinfo`'s policy. + * (`https://host.example#sec@x`, failing closed) are masked under the rest of `urlUserinfoRange`'s policy. * Core's journal projection applies the same predicate (`packages/core/src/tools/bounding.ts`). */ function isPublicQueryContact(userinfo: string, query: number): boolean { @@ -156,7 +187,11 @@ function isPublicQueryContact(userinfo: string, query: number): boolean { const MASKED = '[REDACTED]'; -function maskUrlUserinfo(match: string): string { +/** + * The userinfo range of one finite-scheme URL match, or `undefined` when the policy keeps it: a + * public query contact, a connection string's colonless user or already-masked userinfo. + */ +function urlUserinfoRange(match: string): MaskRange | undefined { const start = match.indexOf('://') + 3; const authority = match.slice(start); const firstQuery = authority.search(/[?#]/); @@ -164,11 +199,11 @@ function maskUrlUserinfo(match: string): string { const at = authorityAt >= 0 ? authorityAt : authority.length - 1; const userinfo = authority.slice(0, at); const query = userinfo.search(/[?#]/); - if (query >= 0 && isPublicQueryContact(userinfo, query)) return match; + if (query >= 0 && isPublicQueryContact(userinfo, query)) return undefined; const colonless = COLONLESS_USERINFO_SCHEME.test(match.slice(0, start - 3)); const colon = userinfo.indexOf(':'); // A connection string's colonless userinfo names a user (`ssh://git@host`), not a credential. - if (colon < 0 && !colonless) return match; + if (colon < 0 && !colonless) return undefined; // Already-masked userinfo stays as it is: an exactly masked password under a connection-string // scheme, whose usernames are public, or a masked or empty username beside it under any scheme. // The authored-content detector's projection `postgres://app:[REDACTED]@db` is a fixed point. @@ -178,15 +213,47 @@ function maskUrlUserinfo(match: string): string { userinfo.slice(colon + 1) === MASKED && (!colonless || username === '' || username === MASKED) ) - return match; - return `${match.slice(0, start)}${MASKED}${authority.slice(at)}`; + return undefined; + return { start, end: start + at }; +} + +/** Every masked range of one rule's matches in `text`. */ +function ruleRanges(text: string, rule: SecretRule): MaskRange[] { + const ranges: MaskRange[] = []; + for (const match of text.matchAll(rule.pattern)) { + const range = rule.mask(match); + if (range !== undefined) { + ranges.push({ start: match.index + range.start, end: match.index + range.end }); + } + } + return ranges; +} + +/** Replace each run of overlapping ranges with one `[REDACTED]`; the rest of `text` is kept. */ +function maskRanges(text: string, ranges: readonly MaskRange[]): string { + const ordered = [...ranges].sort( + (left, right) => left.start - right.start || left.end - right.end, + ); + let output = ''; + let cursor = 0; + let open: { start: number; end: number } | undefined; + for (const range of ordered) { + if (open !== undefined && range.start < open.end) { + open.end = Math.max(open.end, range.end); + continue; + } + if (open !== undefined) { + output += text.slice(cursor, open.start) + MASKED; + cursor = open.end; + } + open = { start: range.start, end: range.end }; + } + if (open === undefined) return text; + return output + text.slice(cursor, open.start) + MASKED + text.slice(open.end); } function applyRule(text: string, rule: SecretRule): string { - const replacement = rule.replacement; - return typeof replacement === 'string' - ? text.replace(rule.pattern, replacement) - : text.replace(rule.pattern, (match) => replacement(match)); + return maskRanges(text, ruleRanges(text, rule)); } /** @@ -257,7 +324,7 @@ function hasSplitCredential(text: string, joined: string, rule: SecretRule): boo if ( originalOffset - start > joinedOffset - joinedStart && // A malformed URL candidate may actually be a public contact query. - (typeof rule.replacement !== 'function' || rule.replacement(match[0]) !== match[0]) + (rule.credential === undefined || rule.credential(match[0])) ) return true; continue; @@ -287,7 +354,11 @@ function scrubNonQuerySecrets(text: string): string { if (applyRule(residual, rule) !== residual) return '[REDACTED]'; } } - return NON_QUERY_SECRET_RULES.reduce((masked, rule) => applyRule(masked, rule), text); + // Every rule's ranges come from the original text and are merged, so none erases another's context. + return maskRanges( + text, + NON_QUERY_SECRET_RULES.flatMap((rule) => ruleRanges(text, rule)), + ); } /** Exact literal query names, applied last so a value cannot erase another family's evidence. */ diff --git a/tools/causal-controls/provenance.test.mjs b/tools/causal-controls/provenance.test.mjs index e2ffd834..869f26f1 100644 --- a/tools/causal-controls/provenance.test.mjs +++ b/tools/causal-controls/provenance.test.mjs @@ -267,10 +267,10 @@ test('every declared filesystem and secrets anchor matches current production so }); test('secrets workload corrections retain independent complete-count and reader-witness pins', () => { - assert.equal(secretsControls.length, 79); + assert.equal(secretsControls.length, 92); assert.equal( secretsControls.reduce((count, control) => count + 1 + (control.witnesses ?? []).length, 0), - 441, + 462, ); for (const [id, file, expectedTests] of [ ['run-default-secret-reader', 'apps/cli/src/commands/run.test.ts', 84], @@ -333,13 +333,26 @@ test('secrets workload corrections retain independent complete-count and reader- 'diagnostic-userinfo-host-length', 'diagnostic-userinfo-host-anchors', 'diagnostic-header-label-bound', + 'diagnostic-header-concat-bound', + 'diagnostic-label-concat-bound', 'diagnostic-header-token-not-label', - 'diagnostic-header-token-not-masked', + 'diagnostic-header-not-nested', + 'diagnostic-header-nested-stop', + 'diagnostic-header-lookbehind', + 'diagnostic-header-escaped-separator', + 'diagnostic-header-quote-run', + 'diagnostic-header-backslash-first', + 'diagnostic-header-escaped-stop', 'diagnostic-bearer-token-not-label', 'diagnostic-basic-token-not-label', - 'diagnostic-bearer-token-not-masked', - 'diagnostic-basic-token-not-masked', - ].map((id) => [id, ['packages/llm/src/scrub-secrets.test.ts'], 191]), + 'diagnostic-label-quoted-label', + 'diagnostic-label-glued-label', + 'diagnostic-label-embedded-stop', + 'diagnostic-label-masked-tail', + 'diagnostic-merged-masking', + 'diagnostic-mask-merge', + 'diagnostic-token-only-mask', + ].map((id) => [id, ['packages/llm/src/scrub-secrets.test.ts'], 198]), ...['diagnostic-inline-projection-order', 'diagnostic-original-control-boundary'].map((id) => [ id, ['apps/cli/src/render/sanitize.test.ts', 'apps/cli/src/process/render-error.test.ts'], diff --git a/tools/secrets-controls/README.md b/tools/secrets-controls/README.md index f36bab4e..3593e232 100644 --- a/tools/secrets-controls/README.md +++ b/tools/secrets-controls/README.md @@ -4,7 +4,7 @@ Step 4 first-run input matrix, Step 5 snapshot/export, Step 7 provider-header and PR #95 B2 redaction controls on physical source copies. `--mechanism ` supports diagnosis only; selected runs retain `complete: false`. The current manifest -declares **79 independent controls and 441 required named assertion witnesses**. +declares **92 independent controls and 462 required named assertion witnesses**. These counts describe the workload; a declaration is not evidence that its removal passed. Actual fixed/removal/restored results belong to the coordinator's summary. @@ -178,7 +178,7 @@ check, split scheme labels, split API-header labels and raw `?`/`#` password rec independently. Four LLM controls remove the before-masking LF/TAB probe, its original left-boundary check, split API-header recognition and raw-password recognition. Every core phase executes the complete 156-case bounding file; every new LLM phase -executes the complete 191-case scrubber file. Independent ordinary tooling tests pin +executes the complete 198-case scrubber file. Independent ordinary tooling tests pin these inventories, the 278-case overlap cohort, 180-case CLI diagnostic cohort and 18-case command-export cohort. Actual collection still has to match in every phase. @@ -259,14 +259,14 @@ previous-word prefilter and the rule's own boundary predicate; retaining either would leave the tested policy active. Header/key recovery removals still change their original-rule boundary only, and all removals retain the same canonical matchers and bounded raw/joined standalone visitors. Current collection is 25 -complete files / 1,438 cases, with each control's exact workload and named target +complete files / 1,445 cases, with each control's exact workload and named target checked separately. These are updated declarations and author checks, not new physical causal captures or a clean review round. R3 is preserved as NONCLEAN; later corrective acceptance belongs to the owning phase register. Prior selected `7e9096a8` captures retain their older source/workload attribution. -The current 79-control / 237-phase full declaration projects 1,429 retained files -(79 × 18 + 7) under the unchanged 2,000-file / 64-MiB export bounds. This is a capacity +The current 92-control / 276-phase full declaration projects 1,663 retained files +(92 × 18 + 7) under the unchanged 2,000-file / 64-MiB export bounds. This is a capacity calculation, not a measured full capture. Historical 41-control captures retain their original source and cohort attribution; selected B2 or original-boundary recovery captures are not full-family acceptance. @@ -292,13 +292,21 @@ prefix then exhausts V8's RegExp stack, and its witness catches the throw) or it anchors (look-alike prefixes such as `john.doe:s3cret` turn public). The snapshot detector's complete workload is now 11 cases. -The second review round added seven controls and 11 named assertions. One unbounds the header -rule's label run (an admitted-size run of labels then exhausts the RegExp stack); two drop the -header token's guards, so a ninth label becomes the token and the real one leaks, or an -already-masked `Bearer [REDACTED]` stops being a fixed point; four drop the same two guards from -the standalone Bearer and Basic rules, so a stacked label becomes the token or a masked JSON -value loses its closing characters. The host-length controls also witness the exact 260/261 -boundary. +The second review round added seven controls and 11 named assertions: an unbounded header label +run, and the label-as-token and masked-marker guards of the header and standalone rules. The +host-length controls also witness the exact 260/261 boundary. + +The third round replaced sequential rule application with one merged masking pass: every rule +finds its matches in the original text, the masked ranges are merged, and a token rule masks only +its token, so masking an exact `[REDACTED]` is the identity and the three masked-marker guards are +gone. The header and label block now has 23 controls, 13 more than the ten it replaced. Seven +re-anchor the earlier label-run, label-whitespace, label-bound and not-a-label controls; the new +ones restore the sequential pass, disable range merging or mask a whole header match; unbound the +concatenated-label lookaheads (an admitted-size glued run then exhausts the RegExp stack); drop +the header's not-a-nested-header guard, nested-header stop or zero-width boundary; weaken its +escaped separator, quote-run prefix, first-character or escaped-stop rules; drop the standalone +rules' closing-quote or glued-label guard or their embedded-label stop; or reintroduce round 2's +masked-marker guard, which exposes a tail glued to the marker. The thin entry point uses the same coordinator, outcome and bounded process join as the filesystem family in [causal-controls](../causal-controls/README.md). diff --git a/tools/secrets-controls/mutations.mjs b/tools/secrets-controls/mutations.mjs index 31f863eb..7a0db6a2 100644 --- a/tools/secrets-controls/mutations.mjs +++ b/tools/secrets-controls/mutations.mjs @@ -98,16 +98,52 @@ const headerSchemeLabelCases = [ 'masks the token after more than one whitespace character following a scheme label', 'masks quoted and escaped tokens beside header scheme labels', ]; -const schemeLabel = String.raw`(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)`; -const notLabel = String.raw`(?![\\"']*${schemeLabel}(?:\s|$))`; -const notMasked = String.raw`(?![\\"']*\[REDACTED\](?:[\s"',;}]|$))`; -const headerLabelValue = String.raw`(?:[\\"']*${schemeLabel}\s+){0,8}${notLabel}${notMasked}(?:\\*["'])?[^\s"',;}]+/gi,`; -const headerLine = (value) => - String.raw` /(^|[^A-Za-z0-9_-])((?:x-)?api-key|x-goog-api-key)(["']?\s*:\s*["']?)` + value; -const labelLine = (first, value) => String.raw` /\b${first}\s+` + value; +const schemeLabel = String.raw`B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c`; +const headerName = String.raw`(?:(?:x-)?api-key|x-goog-api-key)\\*["']?\s*:`; +// The header rule, piece by piece, so a control can remove or weaken exactly one guard. +const header = { + boundary: String.raw`(?[^\s"',;}\\][^\s"',;}]*?)`, + stop: String.raw`(?=(?<=[^A-Za-z0-9_-])${headerName}|\\{0,15}[\s"',;}]|\\{0,15}$)`, +}; +const headerLine = (part = {}) => { + const value = { ...header, ...part }; + return ( + String.raw` /${value.boundary}(?:(?:x-)?api-key|x-goog-api-key)${value.separator}${value.labelRun}` + + String.raw`${value.notLabel}${value.prefix}${value.notNested}${value.token}${value.stop}/dgi,` + ); +}; +// The two standalone label rules share one tail after their label. +const label = { + close: String.raw`[\\"']*\s+`, + notLabel: String.raw`(?![^\w\s]*(?:(?:${schemeLabel})[\\"']*){1,8}(?:\s|$))`, + lead: String.raw`[\\"']*`, + token: String.raw`(?(?=[^\s\\"'])\S*?[^\s\\"',;}])`, + trail: String.raw`[\\"',;}]*`, + stop: String.raw`(?=(?<=[^\w])(?:${schemeLabel})[\\"']*(?:\s|$)|\s|$)`, +}; +const labelLine = (first, part = {}) => { + const value = { ...label, ...part }; + return ( + String.raw` /\b${first}${value.close}${value.notLabel}${value.lead}${value.token}` + + String.raw`${value.trail}${value.stop}/dgi,` + ); +}; const bearerFirst = String.raw`B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r`; const basicFirst = String.raw`B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c`; -const labelToken = String.raw`${notLabel}${notMasked}\S+/gi,`; +const mergedMasking = [ + ' return maskRanges(', + ' text,', + ' NON_QUERY_SECRET_RULES.flatMap((rule) => ruleRanges(text, rule)),', + ' );', +].join('\n'); +const overlapMerge = ' if (open !== undefined && range.start < open.end) {'; +const headerTokenMask = ' mask: tokenGroup,\n joinedPattern:'; const stackedLabelCase = 'masks a header token after up to eight stacked labels and fails closed beyond them'; const standaloneStackCase = 'masks the token after any number of stacked standalone labels'; @@ -308,6 +344,8 @@ function coreProjectionControl(id, before, after, targets, marker = coreProjecti targets.slice(1).map((target) => ({ target, marker })), ); } +/** The complete scrubber test file every LLM diagnostic control runs. */ +const LLM_SCRUB_CASES = 198; function diagnosticControl(id, before, after, targets, marker = diagnosticMarker) { return control( id, @@ -315,7 +353,7 @@ function diagnosticControl(id, before, after, targets, marker = diagnosticMarker before, after, ['packages/llm/src/scrub-secrets.test.ts'], - 191, + LLM_SCRUB_CASES, targets[0], marker, targets.slice(1).map((target) => ({ target, marker })), @@ -1042,7 +1080,7 @@ export const mutations = [ ), diagnosticControl( 'diagnostic-split-header-label', - String.raw`joinedPattern: /(?:(?:x-)?api-key|x-goog-api-key)["']?\s*:/gi,`, + String.raw`joinedPattern: /(?:(?:x-)?api-key|x-goog-api-key)\\*["']?\s*:/gi,`, 'joinedPattern: /a^/g,', ['masks each split position in the finite API header names'], ), @@ -1052,81 +1090,178 @@ export const mutations = [ String.raw` /(?:${urlSchemes}):\/\/[^/\s?#]*@/gi;`, [...diagnosticPasswords, ...rawPasswordAssignments], ), - // PR95: a header value's leading Bearer/Basic label is masked with its token. + // PR95: a header value's leading Bearer/Basic labels are skipped and its token is masked. Without + // the label run, the probes see an unmasked label and token and blank the whole diagnostic. diagnosticControl( 'diagnostic-header-scheme-label', - headerLine(headerLabelValue), - headerLine(String.raw`[^\s"',;}]+/gi,`), + headerLine(), + headerLine({ labelRun: '' }), headerSchemeLabelCases, headerSchemeLabelMarker, ), - // PR95 round 1: stacked labels (`Bearer Bearer `) and multi-character label whitespace. control( 'diagnostic-header-stacked-labels', scrubSecretsLeaf, - headerLine(headerLabelValue), - headerLine(headerLabelValue.replace(String.raw`\s+){0,8}`, String.raw`\s+)?`)), + headerLine(), + headerLine({ labelRun: header.labelRun.replace(String.raw`\s+){0,8}`, String.raw`\s+)?`) }), ['packages/llm/src/scrub-secrets.test.ts'], - 191, + LLM_SCRUB_CASES, 'masks the token after stacked or quoted scheme labels in a header value', headerSchemeLabelMarker, [{ target: stackedLabelCase, marker: 'SECRETS_B2_STACKED_LABEL' }], ), diagnosticControl( 'diagnostic-header-label-whitespace', - headerLine(headerLabelValue), - headerLine(headerLabelValue.replace(String.raw`\s+){0,8}`, String.raw`\s){0,8}`)), + headerLine(), + headerLine({ labelRun: header.labelRun.replace(String.raw`\s+){0,8}`, String.raw`\s){0,8}`) }), ['masks the token after more than one whitespace character following a scheme label'], headerSchemeLabelMarker, ), - // PR95 round 2: the label run is bounded, its token is never another label or a masked span, - // and the standalone label rules take no label as their token. + // Every quantified label group is bounded so no admitted-size run can exhaust the RegExp stack. diagnosticControl( 'diagnostic-header-label-bound', - headerLine(headerLabelValue), - headerLine(headerLabelValue.replace(String.raw`\s+){0,8}`, String.raw`\s+)*`)), + headerLine(), + headerLine({ labelRun: header.labelRun.replace(String.raw`\s+){0,8}`, String.raw`\s+)*`) }), ['keeps admitted-size label runs linear and fails closed'], 'SECRETS_B2_HEADER_LABEL_BOUND', ), + diagnosticControl( + 'diagnostic-header-concat-bound', + headerLine(), + headerLine({ notLabel: header.notLabel.replace('{1,8}', '+') }), + ['keeps admitted-size escape and concatenated-label runs linear and total'], + 'SECRETS_B3_ADMITTED_RUNS', + ), + diagnosticControl( + 'diagnostic-label-concat-bound', + labelLine(bearerFirst), + labelLine(bearerFirst, { notLabel: label.notLabel.replace('{1,8}', '+') }), + ['keeps admitted-size escape and concatenated-label runs linear and total'], + 'SECRETS_B3_ADMITTED_RUNS', + ), + // A header token is never a label, a run of labels or a nested header name. diagnosticControl( 'diagnostic-header-token-not-label', - headerLine(headerLabelValue), - headerLine(headerLabelValue.replace(notLabel, '')), - [stackedLabelCase], - 'SECRETS_B2_STACKED_LABEL', + headerLine(), + headerLine({ notLabel: '' }), + ['treats a labels-only header value alike in every context'], + 'SECRETS_B3_LABEL_CONTEXT', + ), + ...[ + ['diagnostic-header-not-nested', { notNested: '' }], + [ + 'diagnostic-header-nested-stop', + { stop: header.stop.replace(String.raw`(?<=[^A-Za-z0-9_-])${headerName}|`, '') }, + ], + ['diagnostic-header-lookbehind', { boundary: '(?:^|[^A-Za-z0-9_-])' }], + ].map(([id, part]) => + diagnosticControl( + id, + headerLine(), + headerLine(part), + ['keeps every header value and label token apart from a nested header name'], + 'SECRETS_B3_NESTED_HEADER', + ), + ), + // Quotes and escapes around a header token, also in JSON-encoded JSON. + ...[ + [ + 'diagnostic-header-escaped-separator', + { separator: header.separator.replace(String.raw`\\*["']?\s*:`, String.raw`["']?\s*:`) }, + ], + ['diagnostic-header-quote-run', { prefix: String.raw`(?:\\*["'])?` }], + [ + 'diagnostic-header-backslash-first', + { token: header.token.replace(String.raw`[^\s"',;}\\]`, String.raw`[^\s"',;}]`) }, + ], + ].map(([id, part]) => + diagnosticControl( + id, + headerLine(), + headerLine(part), + ['masks a header token after any run of quotes or escapes, also in JSON-encoded JSON'], + 'SECRETS_B3_HEADER_ESCAPES', + ), ), diagnosticControl( - 'diagnostic-header-token-not-masked', - headerLine(headerLabelValue), - headerLine(headerLabelValue.replace(notMasked, '')), + 'diagnostic-header-escaped-stop', + headerLine(), + headerLine({ + stop: header.stop.replace( + String.raw`|\\{0,15}[\s"',;}]|\\{0,15}$)`, + String.raw`|[\s"',;}]|$)`, + ), + }), [fixedPointCase], 'SECRETS_B2_MASKED_FIXED_POINT', ), + // The standalone label rules: no label as a token, labels closed by a quote, glued labels. + ...[ + ['diagnostic-bearer-token-not-label', bearerFirst], + ['diagnostic-basic-token-not-label', basicFirst], + ].map(([id, first]) => + diagnosticControl( + id, + labelLine(first), + labelLine(first, { notLabel: '' }), + [standaloneStackCase], + 'SECRETS_B2_STACKED_LABEL', + ), + ), diagnosticControl( - 'diagnostic-bearer-token-not-label', - labelLine(bearerFirst, labelToken), - labelLine(bearerFirst, labelToken.replace(notLabel, '')), - [standaloneStackCase], - 'SECRETS_B2_STACKED_LABEL', + 'diagnostic-label-quoted-label', + labelLine(bearerFirst), + labelLine(bearerFirst, { close: String.raw`\s+` }), + ['masks the token after a label closed by its own quote'], + 'SECRETS_B3_QUOTED_LABEL', ), + ...[ + [ + 'diagnostic-label-glued-label', + { notLabel: label.notLabel.replace(String.raw`[^\w\s]*`, String.raw`[\\"']*`) }, + ], + [ + 'diagnostic-label-embedded-stop', + { stop: label.stop.replace(String.raw`(?<=[^\w])(?:${schemeLabel})[\\"']*(?:\s|$)|`, '') }, + ], + ].map(([id, part]) => + diagnosticControl( + id, + labelLine(bearerFirst), + labelLine(bearerFirst, part), + ['masks the token after a label glued behind punctuation or inside another token'], + 'SECRETS_B3_GLUED_LABEL', + ), + ), + // Reintroducing round 2's masked-marker guard exposes a tail glued to the marker again. + diagnosticControl( + 'diagnostic-label-masked-tail', + labelLine(bearerFirst), + labelLine(bearerFirst, { + lead: String.raw`(?![\\"']*\[REDACTED\](?:[\s"',;}]|$))` + label.lead, + }), + ["masks a tail glued to a masked token, including the scrubber's own masks"], + 'SECRETS_B3_MASKED_TAIL', + ), + // Masks come from the original text and are merged; each token rule masks only its token. diagnosticControl( - 'diagnostic-basic-token-not-label', - labelLine(basicFirst, labelToken), - labelLine(basicFirst, labelToken.replace(notLabel, '')), - [standaloneStackCase], - 'SECRETS_B2_STACKED_LABEL', + 'diagnostic-merged-masking', + mergedMasking, + ' return NON_QUERY_SECRET_RULES.reduce((masked, rule) => applyRule(masked, rule), text);', + ['masks the token after a label glued behind punctuation or inside another token'], + 'SECRETS_B3_GLUED_LABEL', ), diagnosticControl( - 'diagnostic-bearer-token-not-masked', - labelLine(bearerFirst, labelToken), - labelLine(bearerFirst, labelToken.replace(notMasked, '')), - [fixedPointCase], - 'SECRETS_B2_MASKED_FIXED_POINT', + 'diagnostic-mask-merge', + overlapMerge, + ' if (open !== undefined && false) {', + headerSchemeLabelCases, + headerSchemeLabelMarker, ), diagnosticControl( - 'diagnostic-basic-token-not-masked', - labelLine(basicFirst, labelToken), - labelLine(basicFirst, labelToken.replace(notMasked, '')), + 'diagnostic-token-only-mask', + headerTokenMask, + ' mask: WHOLE_MATCH,\n joinedPattern:', [fixedPointCase], 'SECRETS_B2_MASKED_FIXED_POINT', ), @@ -1179,7 +1314,7 @@ export const mutations = [ contactAssignment, String.raw` return userinfo.includes('=', query + 1);`, ['packages/llm/src/scrub-secrets.test.ts'], - 191, + LLM_SCRUB_CASES, contactHostCase, userinfoContactMarker, rawPasswordAssignments.map((target) => ({ target, marker: diagnosticMarker })), From b2c50e88ec3270bd5aba23ce9f2c7ceb987153b9 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Tue, 20 Oct 2026 09:31:28 +0300 Subject: [PATCH 13/97] fix(llm): address the fourth PR95 review round Round 4 of the PR95 increment's review (at 6fb7e9d9) confirmed 19 findings across the scrubber, tests, docs, security, property and critic lenses. All are fixed or, for one, documented as a deliberate limit. LLM diagnostic backstop (packages/llm/src/scrub-secrets.ts): - The literal query names are one rule of the merged masking pass: their value ranges come from the original text like every other rule, so a header token cut short at `;` no longer leaves a query value's tail visible (SCRUB-R4-3). - An ambiguous authority (userinfo holding a raw ?/# that is not a public contact) is masked through its end unless a host-shaped name follows its last @, so `https://svc?password=P@ssw0rd123` no longer leaves `ssw0rd123` visible while `https://u:pa?ss@h.example` keeps its host (SEC-R4-1). The host test is length-guarded like the contact prefix. - A run of up to eight concatenated labels (`BearerBearer`) is one label in the header value and in the standalone rule, and ends a token glued behind punctuation; nine concatenated labels blank the diagnostic before any rule runs (SCRUB-R4-1, SEC-R4-2, GEN-R4-1, DOC-R4-1). - The Bearer and Basic rules are one rule whose first label is either label, so neither copy can drift from the other (CRIT-R4-1). - scrubSecrets repeats its merged pass until the output stops changing, at most three passes, and fails closed otherwise, so a second scrub changes nothing (SCRUB-R4-2, GEN-R4-2). - A word glued before `sftp://` still reads as sftp: reading it as the colonless ftp would make every whitespace-joined probe, the CLI's display residual included, blank `connect to sftp://user@host`. It is recorded as a limit and pinned (GEN-R4-4). Core projection (packages/core/src/tools/bounding.ts): - The same ambiguous-authority extent and a bounded fixed point. - A scheme label never takes a URL scheme as its token, so the split pass no longer hides a password behind `token postgres://` (GEN-R4-5). - An LF/TAB right after `://` or before the userinfo's @ belongs to the split userinfo (SEC-R4-3). Tests and controls: - 14 scrubber and 6 core cases cover every finding, including the header `'`/`}` stops, quoted standalone tokens, a range containing a later one, Basic copies of the round-3 cases and admitted-size runs that a quadratic regression would push past the case deadline (T4-1..T4-5). The merge-extent witnesses put a key inside an ambiguous authority's rest: the bounded repeat re-masks a truncated range wherever its rule matches again, so only a tail with no @ left discriminates the extent. - The secrets family has 109 controls (481 targets and witnesses): 19 new, the Bearer/Basic not-a-label pair merged, and the equivalent label concatenation-bound control retired. Edges that only change how much is masked are pinned by tests without a control, keeping the family within the 2,000-file evidence bound. The secrets CI job's allowance becomes 60 minutes, like the filesystem job's. Docs: error-handling.md's backstop policy (masking, URL userinfo, trade-offs, limits; DOC-R4-2 and GEN-R4-3 wording), built-in-tools.md, effect-journal.md's digest note and the controls README. Refs: ADR-0113 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 9 +- apps/cli/src/render/sanitize.ts | 4 +- docs/reference/shared-core/built-in-tools.md | 14 +- docs/reference/shared-core/effect-journal.md | 10 +- docs/standards/error-handling.md | 61 ++-- packages/core/src/tools/bounding.test.ts | 74 ++++- packages/core/src/tools/bounding.ts | 49 +++- packages/llm/src/scrub-secrets.test.ts | 282 ++++++++++++++++++- packages/llm/src/scrub-secrets.ts | 144 ++++++---- tools/causal-controls/provenance.test.mjs | 29 +- tools/ci/require-w8-acceptance.test.mjs | 2 +- tools/secrets-controls/README.md | 44 ++- tools/secrets-controls/mutations.mjs | 260 +++++++++++++---- 13 files changed, 819 insertions(+), 163 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a50a68e..8e375554 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -281,9 +281,12 @@ jobs: secrets-mutations: name: Wave 2 secrets causal controls # The historical 35-control, three-phase capture exceeded the former aggregate budget. - # The current cohort is 57 controls (2026-10-18); this comment does not certify its execution. - # Per-phase, case/hook, output and retained-evidence bounds remain unchanged. - timeout-minutes: 30 + # Wave 3's PR95 increment grows the cohort to 109 controls (2026-10-20): each scrubber + # control runs its eight-MiB-heavy file in three phases (about 24 s locally), so the job + # allowance matches the filesystem job's 60 minutes. Per-phase, case/hook, output and + # retained-evidence bounds remain unchanged (109 × 18 + 7 = 1,969 of 2,000 files); + # this comment does not certify execution. + timeout-minutes: 60 runs-on: ubuntu-latest steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 diff --git a/apps/cli/src/render/sanitize.ts b/apps/cli/src/render/sanitize.ts index 80fd1f66..9f409e96 100644 --- a/apps/cli/src/render/sanitize.ts +++ b/apps/cli/src/render/sanitize.ts @@ -64,8 +64,8 @@ export function sanitizeUntrustedInline(text: string): string { * Coarse on purpose: a boolean cannot locate the * span, and this function's whole posture (see {@link sanitizeUntrusted}) is that the cost of a false * positive is a `[REDACTED]` in a diagnostic, while the cost of a miss is a printed key. This broader CLI - * probe also checks split literal query names. The shared `scrubSecrets` separately protects its finite - * non-query credential shapes before query masking can destroy their identifying context; it does not + * probe also checks split literal query names. The shared `scrubSecrets` masks its finite shapes and + * its literal query values in one merged pass, so no mask erases another shape's context; it does not * join query names or ordinary query prose. */ function redactIncludingAcrossSeparators(normalized: string): string { diff --git a/docs/reference/shared-core/built-in-tools.md b/docs/reference/shared-core/built-in-tools.md index 7c29305c..e386b4f6 100644 --- a/docs/reference/shared-core/built-in-tools.md +++ b/docs/reference/shared-core/built-in-tools.md @@ -96,8 +96,12 @@ existing content-sensitive handling. A raw `?` or `#` before a URL authority's `@` begins public query or fragment text only when the text before it is host-shaped (a dotted name or IPv4, `localhost` or a bracketed IP literal, with an optional port) **and** an `=` assignment follows it; otherwise the userinfo is masked. -This is the same predicate as the [LLM diagnostic backstop](../../standards/error-handling.md#diagnostic-secret-backstop), -including its 260-character bound on the host-shaped prefix. The projection otherwise differs from +Masked userinfo that holds such a raw `?` or `#` leaves the authority ambiguous: unless a +host-shaped name follows its last `@`, the text there may be the rest of a query value +(`https://svc?password=P@ss`), so the mask runs through the authority's end, the next `/`, +whitespace, quote, `,` or `}`. This is the same predicate and extent as the +[LLM diagnostic backstop](../../standards/error-handling.md#diagnostic-secret-backstop), including +its 260-character bound on a host-shaped name. The projection otherwise differs from that backstop by design: its authority also ends at `"`, `'`, `,`, `}` and `]`, so userinfo that contains one of them — an already-masked `[REDACTED]` password among them — is left as it is; it recognizes any scheme of up to 32 characters rather than a finite list; and a userinfo with no @@ -105,7 +109,11 @@ password is kept as a vanity handle below 16 characters and masked from 16 on, u LF/TAB inside a recognized token-shaped credential extends the masked original span. An ambiguous token-alphabet continuation after LF/TAB is conservatively masked with -that credential. Ordinary spaces and punctuation separate public prose; the masker +that credential. An LF or TAB right after a URL's `://` or right before its userinfo's `@` +belongs to the split userinfo, which is masked as one span. An authentication scheme label +never takes a URL scheme as its token: `token postgres://app:pw@db` leaves the URL to the +userinfo rule. The projection repeats until its output stops changing, at most three passes, +and becomes `[redacted]` if it has not settled, so projecting a projection changes nothing. Ordinary spaces and punctuation separate public prose; the masker does not discard the whole diagnostic because a credential appears. Supported split shapes are recognized **before** complete-text one-line normalization, followed by ordinary recognition and length clipping. This preserves multiline quoted-value diff --git a/docs/reference/shared-core/effect-journal.md b/docs/reference/shared-core/effect-journal.md index 3d41da84..95c2a18b 100644 --- a/docs/reference/shared-core/effect-journal.md +++ b/docs/reference/shared-core/effect-journal.md @@ -370,7 +370,8 @@ inline-media redaction remain deliberate projection equality; raw credential byt added to the digest to distinguish calls. Recognized URL userinfo is masked through the authority's last `@`, preserving the host, -path, query and fragment. Recognized opaque URL-query values are masked through their delimiter, including values +path, query and fragment; userinfo holding a raw `?` or `#` with no host-shaped name after +that `@` is masked through the authority's end. Recognized opaque URL-query values are masked through their delimiter, including values longer than 512 characters. Quoted secret-key values honor escaped quotes and backslashes through the actual closing quote. A recognized value must not leave a credential suffix in the projection. This remains a finite shape detector, not a claim to identify arbitrary @@ -378,8 +379,11 @@ unlabelled secrets. Existing old digests are not retroactively erased or rewritt A raw `?` or `#` before the authority's `@` is public only after a host-shaped prefix that an `=` assignment follows ([built-in tools](built-in-tools.md#finite-diagnostic-credential-masking)). -Wave 3's PR95 increment tightened that predicate without a new digest domain. It changes a -`v5:` projection only where such userinfo was previously treated as public: a resumed run whose +Wave 3's PR95 increment tightened that predicate without a new digest domain. It also masks +an ambiguous authority through its end, masks userinfo split by an LF or TAB at either edge, +keeps an authentication scheme label from taking a URL scheme as its token, and repeats the +projection until it is stable. These change a `v5:` projection only for the arguments they +project differently, such as userinfo previously treated as public: a resumed run whose committed effect carried one now computes a different digest and refuses matching prepare instead of replaying, which is the fail-closed outcome of any digest difference. Every other `v5:` row keeps its digest. diff --git a/docs/standards/error-handling.md b/docs/standards/error-handling.md index eeb5efdb..40f80fa7 100644 --- a/docs/standards/error-handling.md +++ b/docs/standards/error-handling.md @@ -136,23 +136,31 @@ content. `Bearer Bearer` is a common misconfiguration. Its token may begin with any run of (escaped) quotes and ends at a delimiter, at escapes before one, or before a nested header name, which then masks its own value. A label may be closed by its own quote before its token - (`"Bearer" `). No label takes another label as its token, even one glued behind - punctuation, quoted or concatenated, so a standalone value with any number of stacked labels + (`"Bearer" `), and a run of up to eight concatenated labels (`BearerBearer`) counts as + one label. No label takes another label as its token, even one glued behind punctuation, + quoted or concatenated, so a standalone value with any number of separately stacked labels masks the token after the last one, and a label glued to a token behind a non-word character masks its own token. -- **Masking.** Every rule finds its matches in the original text, and the masked ranges of all - rules are merged before anything is replaced, so no rule can erase the context another rule - needs. A header or label rule masks only its token: the header name, the separator and any - labels stay visible, an LF/TAB-split label keeps its spelling, and quotes around a token stay - outside it. Masking a token that is already exactly `[REDACTED]` changes nothing, so a masked - token is a fixed point, quoted, escaped or after labels; text glued to the marker - (`[REDACTED];tail`) is masked with it. +- **Masking.** Every rule, the literal query names included, finds its matches in the original + text, and the masked ranges of all rules are merged before anything is replaced, so no rule + can erase the context another rule needs. A header, label or query rule masks only its token: + the header name, the separator, any labels and the query name stay visible, an LF/TAB-split + label keeps its spelling, and quotes around a token stay outside it. Masking a token that is + already exactly `[REDACTED]` changes nothing, so a masked token is a fixed point, quoted, + escaped or after labels. After a `Bearer`/`Basic` label, text glued to the marker + (`Bearer [REDACTED];tail`) is masked with it; a header token ends at `,`, `;` or `}`. One + pass can still glue a marker to the text after another rule's token, so the scrub repeats + until its output stops changing, at most three passes, and blanks the whole diagnostic if it + has not settled: a second scrub changes nothing. - **URL userinfo.** The `?`/`#` rule decides where userinfo ends: a raw `?` or `#` before the authority `@` begins public query or fragment text only when the text before it is host-shaped (a dotted name or IPv4, `localhost` or a bracketed IP literal, with an optional port, at most 260 characters in all) **and** an `=` assignment follows it, as in - `https://host.example?contact=a@b.example`. The scheme rule then decides what is masked: - `http(s)`, `ws(s)` and `ftp(s)` mask any userinfo, while the connection-string schemes + `https://host.example?contact=a@b.example`. Masked userinfo that holds such a raw `?` or `#` + leaves the authority ambiguous: unless a host-shaped name follows its last `@`, the text there + may be the rest of a query value (`https://svc?password=P@ss`), so the mask runs through the + authority's end, the next `/`, whitespace, quote, `,` or `}`. The scheme rule then decides + what is masked: `http(s)`, `ws(s)` and `ftp(s)` mask any userinfo, while the connection-string schemes `postgres`, `postgresql`, `mysql`, `mariadb`, `redis`, `rediss`, `mongodb`, `mongodb+srv`, `amqp`, `amqps`, `ssh`, `sftp`, `ldap`, `ldaps`, `smtp` and `smtps` mask only colon-bearing userinfo (`user:pass@`, `:pass@`), so a bare user such as `ssh://git@host` (or a colonless @@ -160,32 +168,41 @@ content. exactly masked password under a connection-string scheme, or every part masked or empty under any scheme. The authored-content detector's projection of `postgres://app:{{secrets.password}}@db` to `postgres://app:[REDACTED]@db` is therefore a fixed point. Core's - journal projection applies the same `?`/`#` predicate + journal projection applies the same `?`/`#` predicate and ambiguous-authority extent ([built-in tools](../reference/shared-core/built-in-tools.md#finite-diagnostic-credential-masking)). - **Split and normalized forms.** Every rule except the literal query names also probes the terminal-normalized text; a credential revealed only by normalization masks the **whole** diagnostic. The key prefixes, URL - userinfo and the `api-key` header label are additionally probed in the LF/TAB-joined view at + userinfo and the exact header names are additionally probed in the LF/TAB-joined view at original-text boundaries, and a credential revealed only by such a split masks the whole diagnostic too. A `Bearer`/`Basic` label may itself be split by LF/TAB. - **Documented trade-offs, all failing closed.** A scheme glued to a word character is masked in place; an unlisted scheme ending in a listed one (`gftp://`, `jdbc:mysql://`) matches by its suffix; prose that the whitespace-stripped residual probe glues to a path-less URL and a later `@` blanks the whole diagnostic; a host fragment with no assignment (`https://host.example#sec@x`), - a single-label host query and an over-long host-shaped prefix are masked; a + a single-label host query and an over-long host-shaped prefix are masked, and so is the text + after the last `@` of such an ambiguous authority unless it is host-shaped; a `Bearer`/`Basic` label that ends a line takes the next token with it; an opaque value after a - label ends only at whitespace, so in compact JSON it masks to the end of the run; and nine or - more labels after a header name, or a token that begins with `,`, `;` or `}` after a header - label, blank the whole diagnostic. Whether nine or more labels with no token after them blank - it can depend on the character before the header name, so a second scrub of such output may - blank what the first kept; no token is visible either way. + label ends only at whitespace, so in compact JSON it masks to the end of the run; nine or more + concatenated labels, and a token that begins with `,`, `;` or `}` after a `Bearer`/`Basic` + label in a header value, blank the whole diagnostic. Past eight separate labels after a header + name, and in a value of labels alone glued behind punctuation (`Bearer |Bearer`), the outcome + depends on what precedes the header name: when its nearest non-whitespace character is a word + character, `-` or `_`, the diagnostic stays in place (the last label masks any token); + otherwise, the start of the text included, it is blanked. No token is visible either way, and + the output is stable under a second scrub. - **Limits.** Userinfo under an unlisted scheme is not masked, including driver-qualified spellings such as `postgresql+psycopg2://`. Opaque `Bearer`/`Basic` values, `api-key` header values and literal secret query values end at any whitespace, LF and TAB included, so a fragment after such a break stays visible, as it does after an ordinary space; a label glued to - a quoted token without whitespace (`Bearer""`) is likewise outside the shape. Malformed URLs - (`https:/u:p@h`), a raw `/` inside a password, invisible characters inside a scheme and JSON - whose quotes are themselves escaped as `\u0022` are outside the finite shapes. + a quoted token without whitespace (`Bearer""`) is likewise outside the shape, as is a + token after a JSON escape (`\n`, `\t`) that stands for the whitespace after a label. A word + glued before `sftp://` (`statusftp://user@h`) reads as `sftp`, so its colonless userinfo stays + visible like any `sftp` user: every whitespace-joined probe, the CLI's display residual among + them, glues prose to a scheme the same way, and reading it as `ftp` would blank an ordinary + `connect to sftp://user@host` diagnostic. Malformed URLs (`https:/u:p@h`), a raw `/` inside a + password, invisible characters inside a scheme and JSON whose quotes are themselves escaped as + `\u0022` are outside the finite shapes. ## No silent catches diff --git a/packages/core/src/tools/bounding.test.ts b/packages/core/src/tools/bounding.test.ts index 3a789333..9d25a034 100644 --- a/packages/core/src/tools/bounding.test.ts +++ b/packages/core/src/tools/bounding.test.ts @@ -1257,14 +1257,84 @@ describe('PR95 raw ?/# userinfo in the journal projection', () => { it('masks an admitted-size dotted userinfo prefix without exhausting the RegExp stack', () => { const labels = 'a.'.repeat(Math.floor((8 * 1024 * 1024 - 256 - 64) / 2)); - const input = `upstream said https://${labels}a?x=1@h failed`; + const input = `upstream said https://${labels}a?x=1@h.example failed`; let output = ''; expect(() => { output = redactSecretShapedText(input); }, 'SECRETS_B2_CORE_USERINFO_HOST_LENGTH').not.toThrow(); expect( - output === 'upstream said https://[redacted]@h failed', + output === 'upstream said https://[redacted]@h.example failed', 'SECRETS_B2_CORE_USERINFO_HOST_LENGTH', ).toBe(true); }, 15_000); }); + +describe('PR95 review round 4: ambiguous authorities, URL schemes after labels and split edges', () => { + it('masks a secret query value that holds a raw @ after a single-label host', () => { + for (const [input, expected, tail] of [ + ['mysql://db?user=root&password=Adm1n@2024!secure', 'mysql://[redacted]', '2024!secure'], + ['https://svc?access_token=ya29@abcdefXYZ', 'https://[redacted]', 'abcdefXYZ'], + ] as const) { + const output = redactSecretShapedText(input); + expect(output, 'SECRETS_B4_CORE_AUTHORITY_REST').toBe(expected); + expect(output.includes(tail), 'SECRETS_B4_CORE_AUTHORITY_REST').toBe(false); + } + // A raw ?/# in a password does not hide the host it connects to. + for (const [input, expected] of [ + ['https://alice:pa?ss@provider.invalid/v1', 'https://[redacted]@provider.invalid/v1'], + ['https://u:p?w@[::1]:8443/x', 'https://[redacted]@[::1]:8443/x'], + ['https://u:p?w@localhost:3000/x', 'https://[redacted]@localhost:3000/x'], + ] as const) { + expect(redactSecretShapedText(input), 'SECRETS_B4_CORE_AUTHORITY_REST').toBe(expected); + } + }); + + it('bounds the host test after an ambiguous authority at admitted size', () => { + const labels = 'a.'.repeat(Math.floor((8 * 1024 * 1024 - 256 - 64) / 2)); + const input = `upstream said https://user:pw?x=1@${labels}a failed`; + let output = ''; + expect(() => { + output = redactSecretShapedText(input); + }, 'SECRETS_B4_CORE_REST_LENGTH').not.toThrow(); + expect( + output === 'upstream said https://[redacted] failed', + 'SECRETS_B4_CORE_REST_LENGTH', + ).toBe(true); + }, 15_000); + + it('never takes a URL scheme as the token after a scheme label', () => { + for (const [input, expected] of [ + [ + 'invalid token postgres://app:hunter2secret@db\n', + 'invalid token postgres://[redacted]@db\n', + ], + ['invalid bearer https://app:hunter2secret@db\n', 'invalid bearer https://[redacted]@db\n'], + ['token postgresql://app:hunter2secret@db', 'token postgresql://[redacted]@db'], + ['token abcdefghijkl\n', 'token [redacted]\n'], + ] as const) { + expect(redactSecretShapedText(input), 'SECRETS_B4_CORE_SCHEME_TOKEN').toBe(expected); + } + }); + + it.each(['\n', '\t'])('masks URL userinfo split at either of its edges (%j)', (separator) => { + for (const input of [ + `https://user:hunter2secret${separator}@host.example/x`, + `https://${separator}user:hunter2secret@host.example/x`, + ]) { + expect(redactSecretShapedText(input), 'SECRETS_B4_CORE_SPLIT_EDGE').toBe( + 'https://[redacted]@host.example/x', + ); + } + }); + + it('reaches a fixed point, failing closed when it cannot', () => { + for (const [input, expected] of [ + ['g://s.hos#mHPGEAXXQ8@?q==a@b.exam/user:3eYPUzfxo', 'g://[redacted]'], + ['[redacted]', '[redacted]'], + ] as const) { + const output = redactSecretShapedText(input); + expect(output, 'SECRETS_B4_CORE_FIXED_POINT').toBe(expected); + expect(redactSecretShapedText(output), 'SECRETS_B4_CORE_FIXED_POINT').toBe(output); + } + }); +}); diff --git a/packages/core/src/tools/bounding.ts b/packages/core/src/tools/bounding.ts index 33ede502..27619e9e 100644 --- a/packages/core/src/tools/bounding.ts +++ b/packages/core/src/tools/bounding.ts @@ -159,8 +159,9 @@ const STANDALONE_CREDENTIAL_PREFIX = new RegExp( // a legitimate boundary or manufacture one inside a public word. The source uses the same fragments. const JOINED_STANDALONE_CREDENTIAL = new RegExp(STANDALONE_CREDENTIAL.source.slice(2), 'y'); +// A token is never a URL scheme: `token postgres://app:pw@db` leaves the URL to the userinfo rule. const SCHEME_CREDENTIAL = - /\b(b[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|b[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c|t[\n\t]*o[\n\t]*k[\n\t]*e[\n\t]*n)\s+(?=[\w.~+/-]{8})[\w.~+/-]+={0,2}/gi; + /\b(b[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|b[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c|t[\n\t]*o[\n\t]*k[\n\t]*e[\n\t]*n)\s+(?=[\w.~+/-]{8})(?![\w.~+/-]*:\/\/)[\w.~+/-]+={0,2}/gi; type SecretSpanVisitor = (start: number, end: number, replacement: string) => void; @@ -299,6 +300,8 @@ function redactJoinedCredentialSpans( text: string, visit: (joined: string, mask: SecretSpanVisitor) => void, boundary?: RegExp, + // URL userinfo owns a separator at either edge: one right after the scheme or before the @ splits it. + ownsEdges = false, ): string { if (!/[\n\t]/.test(text)) return text; const joined = text.replace(/[\n\t]+/g, ''); @@ -318,8 +321,8 @@ function redactJoinedCredentialSpans( return target + removed; } visit(joined, (start, end, replacement) => { - const originalStart = advance(start, true); - const originalEnd = advance(end); + const originalStart = advance(start, !ownsEdges); + const originalEnd = advance(end, ownsEdges); if (boundary?.test(text[originalStart - 1] ?? '')) return; if (!/[\n\t]/.test(text.slice(originalStart, originalEnd))) return; parts.push(text.slice(copied, originalStart), replacement); @@ -332,7 +335,7 @@ function redactJoinedCredentialSpans( function redactSplitCredentials(text: string): string { if (!/[\n\t]/.test(text)) return text; - const urls = redactJoinedCredentialSpans(text, visitUrlUserinfo); + const urls = redactJoinedCredentialSpans(text, visitUrlUserinfo, undefined, true); const keys = redactJoinedCredentialSpans( urls, (joined, mask) => { @@ -363,6 +366,17 @@ function redactSplitCredentials(text: string): string { * no-backtracking-RegExp / no-ReDoS posture. */ export function redactSecretShapedText(text: string): string { + // One pass can change the context a later pass decides on (a masked query value can remove the + // authority's last `@`), so the projection repeats until it is stable within three passes and + // fails closed otherwise: its output is stable under another projection. + const once = redactSecretShapedOnce(text); + if (once === text) return once; + const twice = redactSecretShapedOnce(once); + if (twice === once) return once; + return redactSecretShapedOnce(twice) === twice ? twice : '[redacted]'; +} + +function redactSecretShapedOnce(text: string): string { const projected = redactOpaqueQueryValues(redactUrlUserinfo(redactSplitCredentials(text))) // A PEM private-key block (multi-line, space-separated markers the `private_key` key-pattern can't see). // The body span is bounded (`{0,20000}?`, lazy) so an unterminated block can't drive an unbounded scan. @@ -444,8 +458,9 @@ const HOST_SHAPED_MAX_LENGTH = 260; * tail in the durable digest. Paths and JSON delimiters stop the scan; queries/fragments * after an existing userinfo separator remain public. Before that separator, a raw ?/# * begins public query/fragment text only when the text before it is host-shaped **and** an - * `=` assignment follows it; anything else may be a malformed username or password. This is - * the same predicate as the LLM diagnostic backstop (`packages/llm/src/scrub-secrets.ts`). + * `=` assignment follows it; anything else may be a malformed username or password, and unless a + * host-shaped name follows the last `@` such an ambiguous authority is masked through its end. This + * is the same predicate and extent as the LLM diagnostic backstop (`packages/llm/src/scrub-secrets.ts`). * The projection differs from that backstop by design: its authority also ends at `"`, `'`, `,`, * `}` and `]`, so userinfo containing one of them (an already-masked `[REDACTED]` password among * them) is left as it is; it recognizes any scheme of up to 32 characters rather than a finite list; @@ -459,10 +474,20 @@ function isPublicQueryContact(userinfo: string, query: number): boolean { return HOST_SHAPED_AUTHORITY.test(userinfo.slice(0, query)) && userinfo.includes('=', query + 1); } +/** Whether an authority's text after its last `@` begins with a host, up to any query or fragment. */ +function hostFollows(rest: string): boolean { + const end = rest.search(/[?#]/); + const host = end < 0 ? rest : rest.slice(0, end); + // Bounded like the contact prefix, so an admitted-size run cannot exhaust the RegExp stack. + return host.length <= HOST_SHAPED_MAX_LENGTH && HOST_SHAPED_AUTHORITY.test(host); +} + function visitUrlUserinfo(text: string, mask: SecretSpanVisitor): void { if (!text.includes('://')) return; const scheme = /[a-z][\w+.-]{0,31}:\/\//gi; const delimiter = /[/\s"',}\]]/g; + // The LLM backstop's authority rest: `]` stays inside it, so a bracketed IP host stays whole. + const restDelimiter = /[/\s"',}]/g; for (let match = scheme.exec(text); match !== null; match = scheme.exec(text)) { const start = scheme.lastIndex; delimiter.lastIndex = start; @@ -480,9 +505,17 @@ function visitUrlUserinfo(text: string, mask: SecretSpanVisitor): void { if (query >= 0 && isPublicQueryContact(userinfo, query)) continue; const hasPassword = colon >= 0 && colon < userinfo.length - 1; if (!hasPassword && userinfo.length < 16) continue; - mask(start, start + at, '[redacted]'); + let maskEnd = start + at; + if (query >= 0) { + // A raw ?/# leaves the authority ambiguous: unless a host follows its last `@`, the text there + // may be the rest of a query value (`https://svc?password=P@ss`), so the mask runs through it. + restDelimiter.lastIndex = maskEnd + 1; + const restEnd = restDelimiter.exec(text)?.index ?? text.length; + if (!hostFollows(text.slice(maskEnd + 1, restEnd))) maskEnd = restEnd; + } + mask(start, maskEnd, '[redacted]'); // Skip the recognized userinfo without hiding a later scheme in the public host/tail. - scheme.lastIndex = start + at + 1; + scheme.lastIndex = Math.max(maskEnd, start + at + 1); } } diff --git a/packages/llm/src/scrub-secrets.test.ts b/packages/llm/src/scrub-secrets.test.ts index 371d9986..ef1abde4 100644 --- a/packages/llm/src/scrub-secrets.test.ts +++ b/packages/llm/src/scrub-secrets.test.ts @@ -699,13 +699,13 @@ describe('PR95 review round 1: label stacking, host bounds and masked userinfo', 'masks an admitted-size dotted userinfo prefix without exhausting the RegExp stack', () => { const labels = 'a.'.repeat(Math.floor((8 * 1024 * 1024 - 256 - 64) / 2)); - const input = `upstream said https://${labels}a?x=1@h failed`; + const input = `upstream said https://${labels}a?x=1@h.example failed`; let output = ''; expect(() => { output = scrubSecrets(input); }, 'SECRETS_B2_USERINFO_HOST_LENGTH').not.toThrow(); expect( - output === 'upstream said https://[REDACTED]@h failed', + output === 'upstream said https://[REDACTED]@h.example failed', 'SECRETS_B2_USERINFO_HOST_LENGTH', ).toBe(true); }, @@ -856,6 +856,8 @@ describe('PR95 review round 3: merged masking, glued labels and nested headers', ], ["Authorization: Bearer [REDACTED]'TAILSECRET", 'Authorization: Bearer [REDACTED]'], ['Authorization: Bearer [REDACTED]TAILSECRET', 'Authorization: Bearer [REDACTED]'], + ["Authorization: Basic [REDACTED]'TAILSECRET", 'Authorization: Basic [REDACTED]'], + ['Authorization: Basic [REDACTED]TAILSECRET', 'Authorization: Basic [REDACTED]'], [ '{"authorization":"Bearer sk-proj-abcdefghijklmnopqrst","api_key":"SECRETVALUE77"}', '{"authorization":"Bearer [REDACTED]"}', @@ -878,6 +880,9 @@ describe('PR95 review round 3: merged masking, glued labels and nested headers', ], [`api-key: abc(Bearer ${token}`, 'api-key: [REDACTED] [REDACTED]'], [`Bearer x"Bearer ${token}`, 'Bearer [REDACTED]"Bearer [REDACTED]'], + [`Authorization: Basic x(Basic ${token}`, 'Authorization: Basic [REDACTED]Basic [REDACTED]'], + [`Authorization: Basic (Basic ${token}`, 'Authorization: Basic (Basic [REDACTED]'], + [`Basic x"Basic ${token}`, 'Basic [REDACTED]"Basic [REDACTED]'], // A terminal control inside the run makes the raw and normalized views disagree: fail closed. [`Bearer \u001b[0m""Bearer \n${token}`, '[REDACTED]'], ]); @@ -894,6 +899,8 @@ describe('PR95 review round 3: merged masking, glued labels and nested headers', `Authorization: Basic \\"Bearer\\" ${token}`, 'Authorization: Basic \\"Bearer\\" [REDACTED]', ], + [`Authorization: "Basic" ${token}`, 'Authorization: "Basic" [REDACTED]'], + [`Authorization: Bearer 'Basic' ${token}`, "Authorization: Bearer 'Basic' [REDACTED]"], ]); // A word that only begins with a label is a token; a label at the end of the text has none. expect(scrubSecrets('Authorization: Basic Basicx'), 'SECRETS_B3_QUOTED_LABEL').toBe( @@ -930,7 +937,7 @@ describe('PR95 review round 3: merged masking, glued labels and nested headers', ]); }); - it('treats a labels-only header value alike in every context', () => { + it('treats a whitespace-separated labels-only header value alike in every context', () => { for (const input of ['api-key: Bearer Bearer', 'x api-key: Bearer Bearer', 'api-key: Basic']) { expect(scrubSecrets(input), 'SECRETS_B3_LABEL_CONTEXT').toBe(input); } @@ -946,14 +953,275 @@ describe('PR95 review round 3: merged masking, glued labels and nested headers', const escapes = `api-key: ${'\\\\'.repeat(size)}`; const quoted = `api-key: ${'"Bearer'.repeat(Math.floor(size / 7))} ${token}`; const standalone = `Bearer ${'"Bearer'.repeat(Math.floor(size / 7))} ${token}`; + const basic = `Basic ${'"Basic'.repeat(Math.floor(size / 6))} ${token}`; let results: string[] = []; expect(() => { - results = [escapes, quoted, standalone].map((input) => scrubSecrets(input)); + results = [escapes, quoted, standalone, basic].map((input) => scrubSecrets(input)); }, 'SECRETS_B3_ADMITTED_RUNS').not.toThrow(); - // A run of escapes holds no token, and the token after either glued-label run is masked. + // A run of escapes holds no token, and a glued-label run past eight labels blanks it all. expect(results[0] === escapes, 'SECRETS_B3_ADMITTED_RUNS').toBe(true); - expect(results[1]?.includes(token), 'SECRETS_B3_ADMITTED_RUNS').toBe(false); - expect(results[2]?.includes(token), 'SECRETS_B3_ADMITTED_RUNS').toBe(false); + expect(results.slice(1), 'SECRETS_B3_ADMITTED_RUNS').toEqual(Array(3).fill('[REDACTED]')); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); +}); + +describe('PR95 review round 4: concatenated labels, ambiguous authorities and a fixed point', () => { + const token = ['abcdef', 'ghijkl', '123456'].join(''); + const cases = (marker: string, pairs: readonly (readonly [string, string])[]) => { + for (const [input, expected] of pairs) { + const output = scrubSecrets(input); + expect(output, marker).toBe(expected); + expect(output.includes(token) || output.includes('SECRET'), marker).toBe(false); + expect(scrubSecrets(output), marker).toBe(output); + } + }; + const kept = (marker: string, inputs: readonly string[]) => { + for (const input of inputs) expect(scrubSecrets(input), marker).toBe(input); + }; + + it('masks the token after a concatenated label run in every context', () => { + cases('SECRETS_B4_CONCAT_LABEL', [ + [ + `Authorization: Bearer BearerBearer ${token}`, + 'Authorization: Bearer BearerBearer [REDACTED]', + ], + [`Authorization: BearerBearer ${token}`, 'Authorization: BearerBearer [REDACTED]'], + [`Authorization: BasicBearer ${token}`, 'Authorization: BasicBearer [REDACTED]'], + [`Authorization: Bearer BasicBasic ${token}`, 'Authorization: Bearer BasicBasic [REDACTED]'], + [`Basic bearerbasic ${token}`, 'Basic bearerbasic [REDACTED]'], + [ + `Authorization: Bearer\tBearerBearer ${token}`, + 'Authorization: Bearer\tBearerBearer [REDACTED]', + ], + [`Authorization: B\nearerBasic ${token}`, 'Authorization: B\nearerBasic [REDACTED]'], + [ + `Authorization: Bearer "BearerBearer" ${token}`, + 'Authorization: Bearer "BearerBearer" [REDACTED]', + ], + [ + `{"authorization":"Bearer BearerBearer ${token}"}`, + '{"authorization":"Bearer BearerBearer [REDACTED]"}', + ], + [`api-key: BearerBearer ${token}`, 'api-key: BearerBearer [REDACTED]'], + [`see api-key: BearerBearer ${token}`, 'see api-key: BearerBearer [REDACTED]'], + [`curl -H x-api-key: BearerBearer ${token}`, 'curl -H x-api-key: BearerBearer [REDACTED]'], + [`api-key: BearerBearer [REDACTED]${token}`, 'api-key: BearerBearer [REDACTED]'], + [`x-api-key: }Bearer BearerBearer ${token}`, 'x-api-key: }Bearer BearerBearer [REDACTED]'], + // A token ends before a concatenated run glued to it behind punctuation. + [ + `Authorization: Bearer x(BasicBearer ${token}`, + 'Authorization: Bearer [REDACTED]BasicBearer [REDACTED]', + ], + [ + `Authorization: Basic x(BearerBasic ${token}`, + 'Authorization: Basic [REDACTED]BearerBasic [REDACTED]', + ], + // A terminal control inside the header name reveals the value only once normalized. + [`-H X\u0007-API-KEY: BearerBearer ${token}`, '[REDACTED]'], + ]); + // A concatenated run is never a token: labels alone stay visible. + kept('SECRETS_B4_CONCAT_LABEL', [ + 'Authorization: Bearer BearerBearer', + 'Authorization: Basic BasicBasic', + 'Authorization: BearerBearer', + 'api-key: BearerBearer', + ]); + }); + + it('fails closed on nine concatenated labels and keeps eight in place', () => { + const bound = (value: unknown) => expect(value, 'SECRETS_B4_LABEL_RUN_BOUND'); + for (let count = 1; count <= 12; count += 1) { + for (const label of ['Bearer', 'Basic']) { + const run = label.repeat(count); + bound(scrubSecrets(`Authorization: Bearer ${run} ${token}`)).toBe( + count <= 8 ? `Authorization: Bearer ${run} [REDACTED]` : '[REDACTED]', + ); + bound(scrubSecrets(`api-key: ${run} ${token}`)).toBe( + count <= 8 ? `api-key: ${run} [REDACTED]` : '[REDACTED]', + ); + } + } + // Quotes or escapes between the labels still join them, and a run with no token fails alike. + for (const input of [ + `Authorization: ${'"Bearer'.repeat(9)} ${token}`, + `Authorization: ${'\\"Basic'.repeat(9)} ${token}`, + `Authorization: Bearer ${'Basic'.repeat(9)}`, + ]) { + bound(scrubSecrets(input)).toBe('[REDACTED]'); + } + }); + + it( + 'keeps admitted-size label runs total, concatenated or separate', + () => { + const size = 8 * 1024 * 1024 - 256 - 64; + const concatenated = `Authorization: Bearer ${'Bearer'.repeat(Math.floor(size / 6))} ${token}`; + const separate = `Authorization: ${'Basic '.repeat(Math.floor(size / 6))}${token}`; + let results: string[] = []; + expect(() => { + results = [concatenated, separate].map((input) => scrubSecrets(input)); + }, 'SECRETS_B4_LABEL_RUN_BOUND').not.toThrow(); + expect(results[0] === '[REDACTED]', 'SECRETS_B4_LABEL_RUN_BOUND').toBe(true); + expect( + results[1]?.endsWith('Basic [REDACTED]') === true && !results[1].includes(token), + 'SECRETS_B4_LABEL_RUN_BOUND', + ).toBe(true); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); + + it('ends a header token at a closing quote or brace', () => { + cases('SECRETS_B4_HEADER_STOP', [ + [`{api-key:${token}}`, '{api-key:[REDACTED]}'], + [`api-key: ${token}}`, 'api-key: [REDACTED]}'], + [`{'api-key':'${token}'}`, "{'api-key':'[REDACTED]'}"], + [`{'x-api-key': '${token}', 'b': 1}`, "{'x-api-key': '[REDACTED]', 'b': 1}"], + ]); + }); + + it('masks a quoted standalone token after either label', () => { + cases('SECRETS_B4_LABEL_LEAD', [ + [`Authorization: Bearer "${token}"`, 'Authorization: Bearer "[REDACTED]"'], + [`Authorization: Bearer '${token}'`, "Authorization: Bearer '[REDACTED]'"], + [`{"authorization":"Bearer \\"${token}\\""}`, '{"authorization":"Bearer \\"[REDACTED]\\""}'], + [`Authorization: Basic "${token}"`, 'Authorization: Basic "[REDACTED]"'], + [`Authorization: Basic '${token}'`, "Authorization: Basic '[REDACTED]'"], + [`{"authorization":"Basic \\"${token}\\""}`, '{"authorization":"Basic \\"[REDACTED]\\""}'], + ]); + }); + + it('masks the whole of a range that contains a later one', () => { + // A later pass re-masks a truncated range wherever its rule matches again; behind an ambiguous + // authority no `@` is left, so only the merged extent keeps a key's neighbour masked. + cases('SECRETS_B4_MERGE_EXTENT', [ + ['https://svc?a=b@.sk-abcdefghijklmnopqrstuv.TAILSECRET', 'https://[REDACTED]'], + [ + 'https://svc?a=b@.AIzaSyA1234567890abcdefghijklmn.TAILSECRET failed', + 'https://[REDACTED] failed', + ], + ['{"u":"https://svc#x@.sk-abcdefghijklmnopqrstuv.TAILSECRET"}', '{"u":"https://[REDACTED]"}'], + [ + `https://user:sk-abcdefghijklmnopqrstuv:more${token}@h.example/`, + 'https://[REDACTED]@h.example/', + ], + [`postgres://u:AIzaSyA1234567890abcdefghijklmn:${token}@db/x`, 'postgres://[REDACTED]@db/x'], + ]); + }); + + it('keeps the documented edges of the label and header rules', () => { + cases('SECRETS_B4_RULE_EDGES', [[`api-key : ${token} retry`, 'api-key : [REDACTED] retry']]); + // A label glued to a preceding word character is no label. + kept('SECRETS_B4_RULE_EDGES', [`xBearer ${token}`, `xBasic ${token}`]); + }); + + it('masks a literal query value in the same pass as every other rule', () => { + cases('SECRETS_B4_QUERY_MERGE', [ + [ + '{\\"x-api-key\\":\\"https://h.example/cb?token=abc;SECRETTAIL\\"}', + '{\\"x-api-key\\":\\"[REDACTED]', + ], + ['x-api-key: https://h.example/cb?token=abc;SECRETTAIL', 'x-api-key: [REDACTED]'], + ]); + }); + + it('masks a secret query value that holds a raw @ after a single-label host', () => { + for (const [input, expected, tail] of [ + [ + 'request to https://svc?password=P@ssw0rd123 failed', + 'request to https://[REDACTED] failed', + 'ssw0rd123', + ], + ['https://gateway?api_key=Ab3@Zx9QrT7', 'https://[REDACTED]', 'Zx9QrT7'], + ['postgres://db?options=a:b&password=P@ssw0rd99', 'postgres://[REDACTED]', 'ssw0rd99'], + ['https://svc?token=abc@def&x=1', 'https://[REDACTED]', 'def'], + ] as const) { + const output = scrubSecrets(input); + expect(output, 'SECRETS_B4_AMBIGUOUS_QUERY').toBe(expected); + expect(output.includes(tail), 'SECRETS_B4_AMBIGUOUS_QUERY').toBe(false); + } + }); + + it('masks an ambiguous authority through its end unless a host follows its last @', () => { + cases('SECRETS_B4_AUTHORITY_REST', [ + ['https://svc?opaque=P@SECRETTAIL1 failed', 'https://[REDACTED] failed'], + ['https://svc#frag@SECRETTAIL1 x', 'https://[REDACTED] x'], + ['{"url":"https://svc?code=ab@SECRETTAIL1","n":1}', '{"url":"https://[REDACTED]","n":1}'], + ['https://us?er:TOKEN@h/', 'https://[REDACTED]/'], + ]); + // A raw ?/# in a password does not hide the host it connects to. + for (const [input, expected] of [ + ['https://alice:pa?ss@provider.invalid/v1', 'https://[REDACTED]@provider.invalid/v1'], + ['https://u:p?w@[::1]:8443/x', 'https://[REDACTED]@[::1]:8443/x'], + ['https://u:p?w@localhost:3000/x', 'https://[REDACTED]@localhost:3000/x'], + ] as const) { + expect(scrubSecrets(input), 'SECRETS_B4_AUTHORITY_REST').toBe(expected); + } + }); + + it( + 'bounds the host test after an ambiguous authority at admitted size', + () => { + const labels = 'a.'.repeat(Math.floor((8 * 1024 * 1024 - 256 - 64) / 2)); + const input = `upstream said https://svc?x=1@${labels}a failed`; + let output = ''; + expect(() => { + output = scrubSecrets(input); + }, 'SECRETS_B4_REST_LENGTH').not.toThrow(); + expect(output === 'upstream said https://[REDACTED] failed', 'SECRETS_B4_REST_LENGTH').toBe( + true, + ); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); + + it('reads a word glued before sftp as part of an sftp scheme', () => { + // Every whitespace-joined probe glues prose to a scheme the same way, so `to sftp://` must stay. + kept('SECRETS_B4_GLUED_SFTP', [ + 'statusftp://deploy@h.example', + 'upload to sftp://deploy@files.example failed', + ]); + expect(scrubSecrets('statusftp://deploy:pw@h.example'), 'SECRETS_B4_GLUED_SFTP').toBe( + 'statusftp://[REDACTED]@h.example', + ); + }); + + it('reaches a fixed point, failing closed when it cannot', () => { + for (const [input, expected] of [ + ['api-key: https://"@host.example', 'api-key: [REDACTED]'], + ['{"x-api-key":"https://\\"@h.example"}', '{"x-api-key":"[REDACTED]"}'], + ['x-api-key: Bearer abc"=Bearer xyz', 'x-api-key: Bearer [REDACTED] [REDACTED]'], + ['x-api-key: Bearer abc"(Bearer xyz12345', 'x-api-key: Bearer [REDACTED] [REDACTED]'], + ['api-key:Bearer S":BEAReR', 'api-key:Bearer [REDACTED]'], + ['x:sk-proj-abcdefghijklmnopqrst api-key\\":\\" :', '[REDACTED]'], + ['&key=H X-GOOG-API-KEY:basic |BasiC', '[REDACTED]'], + ['http://i.b#@=\n@', '[REDACTED]'], + ['[REDACTED]', '[REDACTED]'], + ] as const) { + const output = scrubSecrets(input); + expect(output, 'SECRETS_B4_FIXED_POINT').toBe(expected); + expect(scrubSecrets(output), 'SECRETS_B4_FIXED_POINT').toBe(output); + } + }); + + it( + 'keeps quote and escape runs after a label or header token linear at admitted size', + () => { + // Pinned by this case deadline only: a quadratic scan here still finishes, just too late. + const size = 8 * 1024 * 1024 - 256 - 64; + const quotes = '"'.repeat(size); + const inputs = [ + `Bearer ${quotes}`, + `Basic ${quotes}`, + `api-key: ${token}${'\\'.repeat(size)}x`, + ]; + let results: string[] = []; + expect(() => { + results = inputs.map((input) => scrubSecrets(input)); + }, 'SECRETS_B4_ADMITTED_RUNS').not.toThrow(); + expect(results[0] === inputs[0], 'SECRETS_B4_ADMITTED_RUNS').toBe(true); + expect(results[1] === inputs[1], 'SECRETS_B4_ADMITTED_RUNS').toBe(true); + expect(results[2] === 'api-key: [REDACTED]', 'SECRETS_B4_ADMITTED_RUNS').toBe(true); }, ADMITTED_SIZE_CASE_TIMEOUT_MS, ); diff --git a/packages/llm/src/scrub-secrets.ts b/packages/llm/src/scrub-secrets.ts index 84f28cf2..3cbedf83 100644 --- a/packages/llm/src/scrub-secrets.ts +++ b/packages/llm/src/scrub-secrets.ts @@ -2,6 +2,9 @@ import { stripTerminalControls } from '@relavium/shared/terminal-safe'; import { splitProbe, type SplitProbe, type SplitProbeAnchor } from './split-credential-probe.js'; +/** The marker every masked span becomes; a diagnostic that cannot be masked in place becomes it whole. */ +const MASKED = '[REDACTED]'; + /** * Defense-in-depth secret scrub applied at the one `makeLlmError` choke point. The CALLER (adapter) is * still responsible for passing an already-redacted message; this is the **structural backstop** — like @@ -12,15 +15,22 @@ import { splitProbe, type SplitProbe, type SplitProbeAnchor } from './split-cred * other rule can erase its identifying context. A match revealed by normalization masks the whole * diagnostic. The finite patterns check boundaries in the original text (the URL rule has none, like the * pre-PR matcher, so a scheme glued to a word character is masked in place) and preserve already-masked spans; - * this is not universal fragment discovery. Query names are excluded from the residual probe. - * Pure and total for string input: regex matching, one merged masking pass and a finite residual + * this is not universal fragment discovery. Query names are excluded from the residual probe, but their + * values are masked in the same merged pass as every other rule. One pass can glue a marker to the + * text after another rule's token, so the scrub repeats until it reaches a fixed point within three + * passes, failing closed otherwise: its output is stable under another scrub. + * Pure and total for string input: regex matching, merged masking passes and a finite residual * check, with no parsing or external calls on the error path. * Asserted by `llm-error.test.ts` (the backstop) and each adapter's * secret-safety test (a planted secret → a secret-free surfaced `LlmError`); see security-review.md. * The canonical policy is error-handling.md's diagnostic secret backstop. */ export function scrubSecrets(text: string): string { - return scrubQuerySecrets(scrubNonQuerySecrets(text)); + const once = scrubOnce(text); + if (once === text) return once; + const twice = scrubOnce(once); + if (twice === once) return once; + return scrubOnce(twice) === twice ? twice : MASKED; } /** A range of the original diagnostic that one rule masks with `[REDACTED]`. */ @@ -57,13 +67,16 @@ function tokenGroup(match: RegExpExecArray): MaskRange | undefined { * here would let any scheme glued to a preceding word character through unmasked. PR #95 added * ws/wss/ftp/ftps under the same rule, and the PR95 increment added the connection-string schemes * (postgres(ql), mysql, mariadb, redis(s), mongodb(+srv), amqp(s), ssh, sftp, ldap(s), smtp(s)), whose - * colonless userinfo names a user and stays visible (see `urlUserinfoRange`). The rule has no `\b` prefix, - * so it supplies its own joined form (the split-credential scanner would otherwise strip two characters - * that are not there) and its `boundary` blocks no start. Consequences, pinned by tests and fail-closed: an - * unlisted longer scheme name ending in a listed one (`gftp://u@h`) matches by suffix, and the - * whitespace-stripped residual probe turns a path-less URL, prose and a later `@` into a whole-diagnostic - * `[REDACTED]` (the pre-PR matcher did so for http(s); dropping `\b` extends it to prose that stripping glues - * to the scheme). A credential split by a plain SPACE that also holds a raw `@` remains the documented + * colonless userinfo names a user and stays visible (see `urlUserinfoRange`). Only `sftp` ends in another + * listed scheme, the colonless `ftp`, and a word glued before it (`statusftp://`) still reads as `sftp`: + * every whitespace-stripped probe (the residual below, the CLI's display residual) glues ordinary prose + * to a scheme the same way, so reading it as `ftp` would blank `to sftp://user@host`. The rule has no + * `\b` prefix, so it supplies its own joined form (the split-credential scanner would otherwise strip two + * characters that are not there) and its `boundary` blocks no start. Consequences, pinned by tests and + * fail-closed: an unlisted longer scheme name ending in a listed one (`gftp://u@h`) matches by suffix, + * and the whitespace-stripped residual probe turns a path-less URL, prose and a later `@` into a + * whole-diagnostic `[REDACTED]` (the pre-PR matcher did so for http(s); dropping `\b` extends it to + * prose that stripping glues to the scheme). A credential split by a plain SPACE that also holds a raw `@` remains the documented * opaque-fragments-across-spaces limit, as it was for the pre-PR matcher. */ const URL_USERINFO_PATTERN = @@ -105,46 +118,65 @@ const NON_QUERY_SECRET_RULES: readonly SecretRule[] = [ { pattern: URL_USERINFO_PATTERN, joinedPattern: URL_USERINFO_PATTERN, - mask: (match) => urlUserinfoRange(match[0]), - credential: (matched) => urlUserinfoRange(matched) !== undefined, + mask: (match) => urlUserinfoRange(match[0], () => authorityRest(match)), + credential: (matched) => urlUserinfoRange(matched, () => '') !== undefined, boundary: NO_BOUNDARY, anchor: { text: '://', window: URL_SCHEME_WINDOW }, }, // Exact literal header names, also in quoted or JSON-escaped diagnostic objects. No arbitrary // header discovery. A value may begin with up to eight Bearer/Basic labels (possibly quoted, - // escaped, glued behind punctuation or LF/TAB-split), as a doubled `Bearer Bearer` - // misconfiguration does; the token after them is masked and the labels stay. The token may begin - // with any run of (escaped) quotes, is never a label, a run of labels or another header name, and - // ends at a delimiter, at escapes before one or before a header name glued to it, which then masks - // its own value. The label repetition is bounded so no label run can exhaust the RegExp stack; - // past the bound the residual probe blanks the whole diagnostic. + // escaped, glued behind punctuation, LF/TAB-split or a concatenated run of up to eight, as in + // `BearerBearer`), as a doubled `Bearer Bearer` misconfiguration does; the token after them is + // masked and the labels stay. The token may begin with any run of (escaped) quotes, is never a + // label, a run of labels or another header name, and ends at a delimiter, at escapes before one or + // before a header name glued to it, which then masks its own value. Every label repetition is + // bounded so no label run can exhaust the RegExp stack: past eight separate labels the standalone + // label rule masks the token after the last one, and nine concatenated labels blank the diagnostic + // (`LABEL_RUN_PAST_BOUND`). { pattern: - /(?[^\s"',;}\\][^\s"',;}]*?)(?=(?<=[^A-Za-z0-9_-])(?:(?:x-)?api-key|x-goog-api-key)\\*["']?\s*:|\\{0,15}[\s"',;}]|\\{0,15}$)/dgi, + /(?[^\s"',;}\\][^\s"',;}]*?)(?=(?<=[^A-Za-z0-9_-])(?:(?:x-)?api-key|x-goog-api-key)\\*["']?\s*:|\\{0,15}[\s"',;}]|\\{0,15}$)/dgi, mask: tokenGroup, joinedPattern: /(?:(?:x-)?api-key|x-goog-api-key)\\*["']?\s*:/gi, boundary: /[A-Za-z0-9_-]/, }, - // LF/TAB may split the finite scheme LABEL, and a label may be closed by its own quote before the - // whitespace (`"Bearer" `). Preserve the required label/value separator and ordinary - // following prose; opaque value fragments cannot be distinguished from that prose. A label never - // takes another label as its token, even one glued behind punctuation, quoted or concatenated, so - // any number of stacked labels leaves the last one to mask the token, and a token ends before a - // label glued to it behind a non-word character, which then masks its own token. Quotes and - // `,;}` around a token stay outside it unless it continues past them, so `[REDACTED];tail` is - // masked whole. - { - pattern: - /\bB[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r[\\"']*\s+(?![^\w\s]*(?:(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)[\\"']*){1,8}(?:\s|$))[\\"']*(?(?=[^\s\\"'])\S*?[^\s\\"',;}])[\\"',;}]*(?=(?<=[^\w])(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)[\\"']*(?:\s|$)|\s|$)/dgi, - mask: tokenGroup, - }, + // One standalone rule for both labels, so neither can drift from the other. LF/TAB may split the + // finite scheme LABEL, a label may be closed by its own quote before the whitespace + // (`"Bearer" `), and a run of up to eight concatenated labels (`BearerBearer`) counts as one + // label. Preserve the required label/value separator and ordinary following prose; opaque value + // fragments cannot be distinguished from that prose. A label never takes another label as its + // token, even one glued behind punctuation, quoted or concatenated, so any number of stacked labels + // leaves the last one to mask the token, and a token ends before a label (or a concatenated run) + // glued to it behind a non-word character, which then masks its own token. Quotes and `,;}` + // around a token stay outside it unless it continues past them, so `[REDACTED];tail` is masked + // whole. { pattern: - /\bB[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c[\\"']*\s+(?![^\w\s]*(?:(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)[\\"']*){1,8}(?:\s|$))[\\"']*(?(?=[^\s\\"'])\S*?[^\s\\"',;}])[\\"',;}]*(?=(?<=[^\w])(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)[\\"']*(?:\s|$)|\s|$)/dgi, + /\b(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)(?:[\\"']*(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)){0,7}[\\"']*\s+(?![^\w\s]*(?:(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)[\\"']*){1,8}(?:\s|$))[\\"']*(?(?=[^\s\\"'])\S*?[^\s\\"',;}])[\\"',;}]*(?=(?<=[^\w])(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)(?:[\\"']*(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)){0,7}[\\"']*(?:\s|$)|\s|$)/dgi, mask: tokenGroup, }, ]; +/** + * Exact literal query names with exact boundaries, so benign suffixes stay; no URL-name decoding. + * Their values join the merged masking pass, but no probe joins query names or ordinary query prose. + */ +const QUERY_SECRET_RULE: SecretRule = { + pattern: + /[?&](?:(?:x[-_])?api[-_]?key|api[-_]?secret|(?:access|secret)[-_]?key|(?:access|refresh|id|auth|session)[-_]?token|client[-_]?secret|key|token|auth|password|secret)=(?[^&\s#]+)/dgi, + mask: tokenGroup, +}; + +/** The merged pass masks the ranges of every rule; the probes run only over the non-query rules. */ +const SECRET_RULES: readonly SecretRule[] = [...NON_QUERY_SECRET_RULES, QUERY_SECRET_RULE]; + +/** + * Nine concatenated labels (`BearerBearer…`, possibly with quotes or escapes between them) exceed + * every bounded label run above, so the token after them cannot be told from them: fail closed. + */ +const LABEL_RUN_PAST_BOUND = + /(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)(?:[\\"']*(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)){8}/i; + /** * Each boundary-bearing rule compiled once for the split scanner. `splitProbe` checks the scanner's * two invariants here, at module load, so a misconfigured rule fails fast with a typed error instead of @@ -185,13 +217,28 @@ function isPublicQueryContact(userinfo: string, query: number): boolean { return HOST_SHAPED_AUTHORITY.test(userinfo.slice(0, query)) && userinfo.includes('=', query + 1); } -const MASKED = '[REDACTED]'; +/** An authority's text after its last `@`, up to a path, whitespace, a quote, `,` or `}`, as in core. */ +const AUTHORITY_REST = /[^/\s"',}]*/y; + +function authorityRest(match: RegExpExecArray): string { + AUTHORITY_REST.lastIndex = match.index + match[0].length; + return AUTHORITY_REST.exec(match.input)?.[0] ?? ''; +} + +/** Whether an authority's text after its last `@` begins with a host, up to any query or fragment. */ +function hostFollows(rest: string): boolean { + const end = rest.search(/[?#]/); + const host = end < 0 ? rest : rest.slice(0, end); + // Bounded like the contact prefix, so an admitted-size run cannot exhaust the RegExp stack. + return host.length <= HOST_SHAPED_MAX_LENGTH && HOST_SHAPED_AUTHORITY.test(host); +} /** * The userinfo range of one finite-scheme URL match, or `undefined` when the policy keeps it: a - * public query contact, a connection string's colonless user or already-masked userinfo. + * public query contact, a connection string's colonless user or already-masked userinfo. `rest` + * reads the authority after the match. */ -function urlUserinfoRange(match: string): MaskRange | undefined { +function urlUserinfoRange(match: string, rest: () => string): MaskRange | undefined { const start = match.indexOf('://') + 3; const authority = match.slice(start); const firstQuery = authority.search(/[?#]/); @@ -214,7 +261,11 @@ function urlUserinfoRange(match: string): MaskRange | undefined { (!colonless || username === '' || username === MASKED) ) return undefined; - return { start, end: start + at }; + if (query < 0) return { start, end: start + at }; + // A raw ?/# leaves the authority ambiguous: unless a host follows its last `@`, the text there may + // be the rest of a query value (`https://svc?password=P@ss`), so the mask runs through it. + const tail = rest(); + return { start, end: start + at + (hostFollows(tail) ? 0 : 1 + tail.length) }; } /** Every masked range of one rule's matches in `text`. */ @@ -336,8 +387,14 @@ function hasSplitCredential(text: string, joined: string, rule: SecretRule): boo return false; } -function scrubNonQuerySecrets(text: string): string { +/** + * One merged masking pass. Every non-query rule probes the original diagnostic first; then the ranges + * of every rule, the literal query names' included, come from the original text and are merged, so no + * rule erases the context another rule needs. + */ +function scrubOnce(text: string): string { const normalized = normalizeSecretProbe(text); + if (LABEL_RUN_PAST_BOUND.test(normalized)) return MASKED; const joined = /[\n\t]/.test(normalized) ? normalized.replace(/[\n\t]+/g, '') : undefined; for (const rule of NON_QUERY_SECRET_RULES) { const rawMasked = applyRule(text, rule); @@ -354,17 +411,8 @@ function scrubNonQuerySecrets(text: string): string { if (applyRule(residual, rule) !== residual) return '[REDACTED]'; } } - // Every rule's ranges come from the original text and are merged, so none erases another's context. return maskRanges( text, - NON_QUERY_SECRET_RULES.flatMap((rule) => ruleRanges(text, rule)), + SECRET_RULES.flatMap((rule) => ruleRanges(text, rule)), ); } - -/** Exact literal query names, applied last so a value cannot erase another family's evidence. */ -function scrubQuerySecrets(text: string): string { - return text.replace( - /([?&](?:(?:x[-_])?api[-_]?key|api[-_]?secret|(?:access|secret)[-_]?key|(?:access|refresh|id|auth|session)[-_]?token|client[-_]?secret|key|token|auth|password|secret)=)[^&\s#]+/gi, - '$1[REDACTED]', - ); // Finite literal names; exact boundaries preserve benign suffixes. No URL-name decoding. -} diff --git a/tools/causal-controls/provenance.test.mjs b/tools/causal-controls/provenance.test.mjs index 869f26f1..e49bcb1a 100644 --- a/tools/causal-controls/provenance.test.mjs +++ b/tools/causal-controls/provenance.test.mjs @@ -267,10 +267,10 @@ test('every declared filesystem and secrets anchor matches current production so }); test('secrets workload corrections retain independent complete-count and reader-witness pins', () => { - assert.equal(secretsControls.length, 92); + assert.equal(secretsControls.length, 109); assert.equal( secretsControls.reduce((count, control) => count + 1 + (control.witnesses ?? []).length, 0), - 462, + 481, ); for (const [id, file, expectedTests] of [ ['run-default-secret-reader', 'apps/cli/src/commands/run.test.ts', 84], @@ -313,7 +313,12 @@ test('secrets workload corrections retain independent complete-count and reader- 'core-userinfo-contact-host', 'core-userinfo-host-length', 'core-userinfo-host-anchors', - ].map((id) => [id, ['packages/core/src/tools/bounding.test.ts'], 156]), + 'core-userinfo-authority-rest', + 'core-userinfo-rest-length', + 'core-scheme-url-lookahead', + 'core-userinfo-split-edges', + 'core-fixed-point', + ].map((id) => [id, ['packages/core/src/tools/bounding.test.ts'], 162]), ...[ 'diagnostic-joined-before-masking', 'diagnostic-joined-original-boundary', @@ -334,7 +339,6 @@ test('secrets workload corrections retain independent complete-count and reader- 'diagnostic-userinfo-host-anchors', 'diagnostic-header-label-bound', 'diagnostic-header-concat-bound', - 'diagnostic-label-concat-bound', 'diagnostic-header-token-not-label', 'diagnostic-header-not-nested', 'diagnostic-header-nested-stop', @@ -343,8 +347,7 @@ test('secrets workload corrections retain independent complete-count and reader- 'diagnostic-header-quote-run', 'diagnostic-header-backslash-first', 'diagnostic-header-escaped-stop', - 'diagnostic-bearer-token-not-label', - 'diagnostic-basic-token-not-label', + 'diagnostic-label-token-not-label', 'diagnostic-label-quoted-label', 'diagnostic-label-glued-label', 'diagnostic-label-embedded-stop', @@ -352,7 +355,19 @@ test('secrets workload corrections retain independent complete-count and reader- 'diagnostic-merged-masking', 'diagnostic-mask-merge', 'diagnostic-token-only-mask', - ].map((id) => [id, ['packages/llm/src/scrub-secrets.test.ts'], 198]), + // Round 4: one label rule for both labels, and each new leak or totality mechanism. + ...['concat-run', 'concat-stop', 'concat-refusal', 'start-bound', 'lead', 'run-bound'].map( + (mechanism) => `diagnostic-label-${mechanism}`, + ), + 'diagnostic-header-concat-run', + 'diagnostic-header-stop-brace', + 'diagnostic-header-stop-quote', + 'diagnostic-query-merge', + 'diagnostic-mask-merge-extent', + 'diagnostic-fixed-point', + 'diagnostic-userinfo-authority-rest', + 'diagnostic-userinfo-rest-length', + ].map((id) => [id, ['packages/llm/src/scrub-secrets.test.ts'], 212]), ...['diagnostic-inline-projection-order', 'diagnostic-original-control-boundary'].map((id) => [ id, ['apps/cli/src/render/sanitize.test.ts', 'apps/cli/src/process/render-error.test.ts'], diff --git a/tools/ci/require-w8-acceptance.test.mjs b/tools/ci/require-w8-acceptance.test.mjs index e2bbd452..2cb1441a 100644 --- a/tools/ci/require-w8-acceptance.test.mjs +++ b/tools/ci/require-w8-acceptance.test.mjs @@ -93,7 +93,7 @@ test('the actual required workflow joins both full offline controls and checks t const job = workflow.jobs[`${kind}-mutations`]; assert.equal(job['continue-on-error'], undefined); assert.equal(job.if, undefined); - assert.equal(job['timeout-minutes'], 30); + assert.equal(job['timeout-minutes'], 60); assert.ok(job.steps.some((step) => step.run === 'pnpm install --frozen-lockfile')); assert.ok(job.steps.some((step) => step.run === 'pnpm test:acceptance-tools')); assert.ok(job.steps.some((step) => step.with?.['node-version-file'] === '.nvmrc')); diff --git a/tools/secrets-controls/README.md b/tools/secrets-controls/README.md index 3593e232..09ec6ece 100644 --- a/tools/secrets-controls/README.md +++ b/tools/secrets-controls/README.md @@ -4,7 +4,7 @@ Step 4 first-run input matrix, Step 5 snapshot/export, Step 7 provider-header and PR #95 B2 redaction controls on physical source copies. `--mechanism ` supports diagnosis only; selected runs retain `complete: false`. The current manifest -declares **92 independent controls and 462 required named assertion witnesses**. +declares **109 independent controls and 481 required named assertion witnesses**. These counts describe the workload; a declaration is not evidence that its removal passed. Actual fixed/removal/restored results belong to the coordinator's summary. @@ -41,10 +41,11 @@ secret suffix after an overlapping query value. The normalized-secret-probe cont removes the canonical `stripTerminalControls` view from the shared non-query probe; 29 named normalized-key overlap witnesses require that view in LLM messages/codes, both CLI sanitizer functions, all four renderer modes and a verbose stack. The -non-query-before-query control reverses the final composition and requires 58 named -Bearer/Basic overlap witnesses across those same surfaces. These controls require -each finite non-query rule to inspect its own original diagnostic views before any -other replacement erases its evidence, followed by the finite query replacement. +non-query-before-query control applies the literal query names before the merged +pass and requires 58 named Bearer/Basic overlap witnesses across those same surfaces. +These controls require each finite non-query rule to inspect its own original +diagnostic views before any other replacement erases its evidence; the literal query +values then join the same merged masking pass instead of following it. The views include the canonical terminal-normalized text and a whitespace-joined residual view. The split literal query name remains an independent CLI obligation. Across the thirteen Step 3 controls, 191 named assertion witnesses are required. The shared @@ -177,8 +178,8 @@ modern `sk-` body characters, joined-span application, the original-text left-bo check, split scheme labels, split API-header labels and raw `?`/`#` password recognition independently. Four LLM controls remove the before-masking LF/TAB probe, its original left-boundary check, split API-header recognition and raw-password recognition. -Every core phase executes the complete 156-case bounding file; every new LLM phase -executes the complete 198-case scrubber file. Independent ordinary tooling tests pin +Every core phase executes the complete 162-case bounding file; every new LLM phase +executes the complete 212-case scrubber file. Independent ordinary tooling tests pin these inventories, the 278-case overlap cohort, 180-case CLI diagnostic cohort and 18-case command-export cohort. Actual collection still has to match in every phase. @@ -259,14 +260,14 @@ previous-word prefilter and the rule's own boundary predicate; retaining either would leave the tested policy active. Header/key recovery removals still change their original-rule boundary only, and all removals retain the same canonical matchers and bounded raw/joined standalone visitors. Current collection is 25 -complete files / 1,445 cases, with each control's exact workload and named target +complete files / 1,465 cases, with each control's exact workload and named target checked separately. These are updated declarations and author checks, not new physical causal captures or a clean review round. R3 is preserved as NONCLEAN; later corrective acceptance belongs to the owning phase register. Prior selected `7e9096a8` captures retain their older source/workload attribution. -The current 92-control / 276-phase full declaration projects 1,663 retained files -(92 × 18 + 7) under the unchanged 2,000-file / 64-MiB export bounds. This is a capacity +The current 109-control / 327-phase full declaration projects 1,969 retained files +(109 × 18 + 7) under the unchanged 2,000-file / 64-MiB export bounds. This is a capacity calculation, not a measured full capture. Historical 41-control captures retain their original source and cohort attribution; selected B2 or original-boundary recovery captures are not full-family acceptance. @@ -308,6 +309,29 @@ escaped separator, quote-run prefix, first-character or escaped-stop rules; drop rules' closing-quote or glued-label guard or their embedded-label stop; or reintroduce round 2's masked-marker guard, which exposes a tail glued to the marker. +The fourth round merged the two standalone label rules into one, whose first label is either +label, so neither copy can drift from the other: the former Bearer and Basic not-a-label controls +became `diagnostic-label-token-not-label`, and every label control now covers both labels. A run +of up to eight concatenated labels (`BearerBearer`) is one label, nine blank the diagnostic, the +literal query names join the merged pass, an ambiguous authority is masked through its end, and +the scrubber repeats its pass to a fixed point. Nineteen controls were added: the label run, its +embedded stop, its not-a-label refusal (`{1,8}` weakened to `{1}`), its stack bound and its +leading quote run; the header's concatenated run and its `}`/`'` stops; the nine-label bound; +the query merge; the merge extent; the fixed point; the ambiguous-authority extent and its host +length bound (with core twins for the last two); and core's URL-scheme lookahead after a scheme +label, its split-userinfo edges and its fixed point. With the merged pair, 109 controls remain: +`diagnostic-label-concat-bound` was retired as an equivalent mutant: the label rule's not-a-label bound is reached only by nine or more +concatenated labels, which the nine-label bound blanks first, or in the whitespace-stripped +residual, where a label rule (which needs whitespace after the label) never matches. The header's +bound stays controlled through the residual of a separate-label run. Edges that only change how +much is masked, never whether a token leaks (a word-glued label is no label, whitespace before a +header colon, a host after an ambiguous authority staying visible), are pinned by tests without a +control, which keeps the family inside the 2,000-file export bound. Two quadratic guards (the label +token's first-character lookahead and the header stop's bounded escape run) and the URL rule's +`://` prefilter are output-equivalent and pinned only by the admitted-size case deadline: the +coordinator counts a timeout as infrastructure, never as a removal's failure, so they have no +control. + The thin entry point uses the same coordinator, outcome and bounded process join as the filesystem family in [causal-controls](../causal-controls/README.md). The provider-header diagnostic control pins the store's fixed throwable message diff --git a/tools/secrets-controls/mutations.mjs b/tools/secrets-controls/mutations.mjs index 7a0db6a2..ba72c9cc 100644 --- a/tools/secrets-controls/mutations.mjs +++ b/tools/secrets-controls/mutations.mjs @@ -104,7 +104,7 @@ const headerName = String.raw`(?:(?:x-)?api-key|x-goog-api-key)\\*["']?\s*:`; const header = { boundary: String.raw`(? { String.raw`${value.notLabel}${value.prefix}${value.notNested}${value.token}${value.stop}/dgi,` ); }; -// The two standalone label rules share one tail after their label. +// The one standalone label rule, piece by piece; its first label is either label. const label = { + first: String.raw`\b(?:${schemeLabel})`, + concat: String.raw`(?:[\\"']*(?:${schemeLabel})){0,7}`, close: String.raw`[\\"']*\s+`, notLabel: String.raw`(?![^\w\s]*(?:(?:${schemeLabel})[\\"']*){1,8}(?:\s|$))`, lead: String.raw`[\\"']*`, token: String.raw`(?(?=[^\s\\"'])\S*?[^\s\\"',;}])`, trail: String.raw`[\\"',;}]*`, - stop: String.raw`(?=(?<=[^\w])(?:${schemeLabel})[\\"']*(?:\s|$)|\s|$)`, + stop: String.raw`(?=(?<=[^\w])(?:${schemeLabel})(?:[\\"']*(?:${schemeLabel})){0,7}[\\"']*(?:\s|$)|\s|$)`, }; -const labelLine = (first, part = {}) => { +const labelLine = (part = {}) => { const value = { ...label, ...part }; return ( - String.raw` /\b${first}${value.close}${value.notLabel}${value.lead}${value.token}` + + String.raw` /${value.first}${value.concat}${value.close}${value.notLabel}${value.lead}${value.token}` + String.raw`${value.trail}${value.stop}/dgi,` ); }; -const bearerFirst = String.raw`B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r`; -const basicFirst = String.raw`B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c`; const mergedMasking = [ ' return maskRanges(', ' text,', - ' NON_QUERY_SECRET_RULES.flatMap((rule) => ruleRanges(text, rule)),', + ' SECRET_RULES.flatMap((rule) => ruleRanges(text, rule)),', ' );', ].join('\n'); +const queryRule = (names) => String.raw`/[?&](?:${names})=(?[^&\s#]+)/dgi`; +const queryNames = String.raw`(?:x[-_])?api[-_]?key|api[-_]?secret|(?:access|secret)[-_]?key|(?:access|refresh|id|auth|session)[-_]?token|client[-_]?secret|key|token|auth|password|secret`; const overlapMerge = ' if (open !== undefined && range.start < open.end) {'; +const mergeExtent = ' open.end = Math.max(open.end, range.end);'; +const hostFollowsLine = + ' return host.length <= HOST_SHAPED_MAX_LENGTH && HOST_SHAPED_AUTHORITY.test(host);'; +const llmFixedPoint = [ + ' const once = scrubOnce(text);', + ' if (once === text) return once;', + ' const twice = scrubOnce(once);', + ' if (twice === once) return once;', + ' return scrubOnce(twice) === twice ? twice : MASKED;', +].join('\n'); +const coreFixedPoint = [ + ' const once = redactSecretShapedOnce(text);', + ' if (once === text) return once;', + ' const twice = redactSecretShapedOnce(once);', + ' if (twice === once) return once;', + " return redactSecretShapedOnce(twice) === twice ? twice : '[redacted]';", +].join('\n'); +const coreSchemeCredential = String.raw` /\b(b[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|b[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c|t[\n\t]*o[\n\t]*k[\n\t]*e[\n\t]*n)\s+(?=[\w.~+/-]{8})(?![\w.~+/-]*:\/\/)[\w.~+/-]+={0,2}/gi;`; const headerTokenMask = ' mask: tokenGroup,\n joinedPattern:'; const stackedLabelCase = 'masks a header token after up to eight stacked labels and fails closed beyond them'; @@ -303,6 +323,8 @@ function redactJoinedCredentialSpans( text: string, visit: (joined: string, mask: SecretSpanVisitor) => void, boundary?: RegExp, + // URL userinfo owns a separator at either edge: one right after the scheme or before the @ splits it. + ownsEdges = false, ): string { if (!/[\n\t]/.test(text)) return text; const joined = text.replace(/[\n\t]+/g, ''); @@ -322,8 +344,8 @@ function redactJoinedCredentialSpans( return target + removed; } visit(joined, (start, end, replacement) => { - const originalStart = advance(start, true); - const originalEnd = advance(end); + const originalStart = advance(start, !ownsEdges); + const originalEnd = advance(end, ownsEdges); if (boundary?.test(text[originalStart - 1] ?? '')) return;`; const withoutCoreOriginalBoundary = coreOriginalBoundary .replace( @@ -331,6 +353,8 @@ const withoutCoreOriginalBoundary = coreOriginalBoundary String.raw` const candidates = /([A-Za-z])/g;`, ) .replace(" if (boundary?.test(text[originalStart - 1] ?? '')) return;", ''); +/** The complete core projection test file every core projection control runs. */ +const CORE_PROJECTION_CASES = 162; function coreProjectionControl(id, before, after, targets, marker = coreProjectionMarker) { return control( id, @@ -338,14 +362,14 @@ function coreProjectionControl(id, before, after, targets, marker = coreProjecti before, after, coreProjectionCases, - 156, + CORE_PROJECTION_CASES, targets[0], marker, targets.slice(1).map((target) => ({ target, marker })), ); } /** The complete scrubber test file every LLM diagnostic control runs. */ -const LLM_SCRUB_CASES = 198; +const LLM_SCRUB_CASES = 212; function diagnosticControl(id, before, after, targets, marker = diagnosticMarker) { return control( id, @@ -461,8 +485,8 @@ export const mutations = [ control( 'query-names', scrubSecretsLeaf, - String.raw`/([?&](?:(?:x[-_])?api[-_]?key|api[-_]?secret|(?:access|secret)[-_]?key|(?:access|refresh|id|auth|session)[-_]?token|client[-_]?secret|key|token|auth|password|secret)=)[^&\s#]+/gi`, - String.raw`/([?&](?:(?:x[-_])?api[-_]?key|key|token|access[-_]?token|auth|password|secret)=)[^&\s#]+/gi`, + queryRule(queryNames), + queryRule(String.raw`(?:x[-_])?api[-_]?key|key|token|access[-_]?token|auth|password|secret`), ['packages/llm/src/llm-error.test.ts'], 98, 'SECRETS_CAUSAL_QUERY_NAMES masks each added family without another credential shape', @@ -577,8 +601,14 @@ export const mutations = [ control( 'non-query-before-query', scrubSecretsLeaf, - ' return scrubQuerySecrets(scrubNonQuerySecrets(text));', - ' return scrubNonQuerySecrets(scrubQuerySecrets(text));', + mergedMasking, + [ + ' const queried = applyRule(text, QUERY_SECRET_RULE);', + ' return maskRanges(', + ' queried,', + ' NON_QUERY_SECRET_RULES.flatMap((rule) => ruleRanges(queried, rule)),', + ' );', + ].join('\n'), overlapFiles, 278, `${overlapMarker} protects client_secret Bearer before query redaction`, @@ -1046,8 +1076,8 @@ export const mutations = [ ), coreProjectionControl( 'core-split-scheme-label', - String.raw`/\b(b[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|b[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c|t[\n\t]*o[\n\t]*k[\n\t]*e[\n\t]*n)\s+(?=[\w.~+/-]{8})[\w.~+/-]+={0,2}/gi`, - String.raw`/\b(bearer|basic|token)\s+(?=[\w.~+/-]{8})[\w.~+/-]+={0,2}/gi`, + coreSchemeCredential, + String.raw` /\b(bearer|basic|token)\s+(?=[\w.~+/-]{8})(?![\w.~+/-]*:\/\/)[\w.~+/-]+={0,2}/gi;`, splitCoreHeaders, ), coreProjectionControl( @@ -1129,22 +1159,64 @@ export const mutations = [ 'diagnostic-header-concat-bound', headerLine(), headerLine({ notLabel: header.notLabel.replace('{1,8}', '+') }), - ['keeps admitted-size escape and concatenated-label runs linear and total'], - 'SECRETS_B3_ADMITTED_RUNS', + ['keeps admitted-size label runs linear and fails closed'], + 'SECRETS_B2_HEADER_LABEL_BOUND', + ), + // Round 4: a run of up to eight concatenated labels is one label, in a header value and alone. + diagnosticControl( + 'diagnostic-header-concat-run', + headerLine(), + headerLine({ + labelRun: header.labelRun.replace(String.raw`(?:[\\"']*(?:${schemeLabel})){0,7}`, ''), + }), + ['masks the token after a concatenated label run in every context'], + 'SECRETS_B4_CONCAT_LABEL', + ), + ...[ + ['diagnostic-label-concat-run', { concat: '' }], + ['diagnostic-label-concat-stop', { stop: label.stop.replace(label.concat, '') }], + ['diagnostic-label-concat-refusal', { notLabel: label.notLabel.replace('{1,8}', '{1}') }], + ].map(([id, part]) => + diagnosticControl( + id, + labelLine(), + labelLine(part), + ['masks the token after a concatenated label run in every context'], + 'SECRETS_B4_CONCAT_LABEL', + ), + ), + // The concatenated run is bounded; the residual of a separate-label run reaches it. + diagnosticControl( + 'diagnostic-label-start-bound', + labelLine(), + labelLine({ concat: label.concat.replace('{0,7}', '*') }), + ['keeps admitted-size label runs linear and fails closed'], + 'SECRETS_B2_HEADER_LABEL_BOUND', + ), + diagnosticControl( + 'diagnostic-label-lead', + labelLine(), + labelLine({ lead: '' }), + ['masks a quoted standalone token after either label'], + 'SECRETS_B4_LABEL_LEAD', ), + // Nine concatenated labels exceed every bounded run, so the diagnostic fails closed. diagnosticControl( - 'diagnostic-label-concat-bound', - labelLine(bearerFirst), - labelLine(bearerFirst, { notLabel: label.notLabel.replace('{1,8}', '+') }), - ['keeps admitted-size escape and concatenated-label runs linear and total'], - 'SECRETS_B3_ADMITTED_RUNS', + 'diagnostic-label-run-bound', + ' if (LABEL_RUN_PAST_BOUND.test(normalized)) return MASKED;', + '', + [ + 'fails closed on nine concatenated labels and keeps eight in place', + 'keeps admitted-size label runs total, concatenated or separate', + ], + 'SECRETS_B4_LABEL_RUN_BOUND', ), // A header token is never a label, a run of labels or a nested header name. diagnosticControl( 'diagnostic-header-token-not-label', headerLine(), headerLine({ notLabel: '' }), - ['treats a labels-only header value alike in every context'], + ['treats a whitespace-separated labels-only header value alike in every context'], 'SECRETS_B3_LABEL_CONTEXT', ), ...[ @@ -1183,6 +1255,18 @@ export const mutations = [ 'SECRETS_B3_HEADER_ESCAPES', ), ), + ...[ + ['diagnostic-header-stop-brace', String.raw`[\s"',;]`], + ['diagnostic-header-stop-quote', String.raw`[\s",;}]`], + ].map(([id, stop]) => + diagnosticControl( + id, + headerLine(), + headerLine({ stop: header.stop.replace(String.raw`[\s"',;}]`, stop) }), + ['ends a header token at a closing quote or brace'], + 'SECRETS_B4_HEADER_STOP', + ), + ), diagnosticControl( 'diagnostic-header-escaped-stop', headerLine(), @@ -1195,23 +1279,18 @@ export const mutations = [ [fixedPointCase], 'SECRETS_B2_MASKED_FIXED_POINT', ), - // The standalone label rules: no label as a token, labels closed by a quote, glued labels. - ...[ - ['diagnostic-bearer-token-not-label', bearerFirst], - ['diagnostic-basic-token-not-label', basicFirst], - ].map(([id, first]) => - diagnosticControl( - id, - labelLine(first), - labelLine(first, { notLabel: '' }), - [standaloneStackCase], - 'SECRETS_B2_STACKED_LABEL', - ), + // The standalone label rule: no label as a token, labels closed by a quote, glued labels. + diagnosticControl( + 'diagnostic-label-token-not-label', + labelLine(), + labelLine({ notLabel: '' }), + [standaloneStackCase], + 'SECRETS_B2_STACKED_LABEL', ), diagnosticControl( 'diagnostic-label-quoted-label', - labelLine(bearerFirst), - labelLine(bearerFirst, { close: String.raw`\s+` }), + labelLine(), + labelLine({ close: String.raw`\s+` }), ['masks the token after a label closed by its own quote'], 'SECRETS_B3_QUOTED_LABEL', ), @@ -1222,13 +1301,18 @@ export const mutations = [ ], [ 'diagnostic-label-embedded-stop', - { stop: label.stop.replace(String.raw`(?<=[^\w])(?:${schemeLabel})[\\"']*(?:\s|$)|`, '') }, + { + stop: label.stop.replace( + String.raw`(?<=[^\w])(?:${schemeLabel})${label.concat}[\\"']*(?:\s|$)|`, + '', + ), + }, ], ].map(([id, part]) => diagnosticControl( id, - labelLine(bearerFirst), - labelLine(bearerFirst, part), + labelLine(), + labelLine(part), ['masks the token after a label glued behind punctuation or inside another token'], 'SECRETS_B3_GLUED_LABEL', ), @@ -1236,8 +1320,8 @@ export const mutations = [ // Reintroducing round 2's masked-marker guard exposes a tail glued to the marker again. diagnosticControl( 'diagnostic-label-masked-tail', - labelLine(bearerFirst), - labelLine(bearerFirst, { + labelLine(), + labelLine({ lead: String.raw`(?![\\"']*\[REDACTED\](?:[\s"',;}]|$))` + label.lead, }), ["masks a tail glued to a masked token, including the scrubber's own masks"], @@ -1247,10 +1331,40 @@ export const mutations = [ diagnosticControl( 'diagnostic-merged-masking', mergedMasking, - ' return NON_QUERY_SECRET_RULES.reduce((masked, rule) => applyRule(masked, rule), text);', + ' return SECRET_RULES.reduce((masked, rule) => applyRule(masked, rule), text);', ['masks the token after a label glued behind punctuation or inside another token'], 'SECRETS_B3_GLUED_LABEL', ), + // Round 4: applying the query names after the merged pass loses a value a header token cut short. + diagnosticControl( + 'diagnostic-query-merge', + mergedMasking, + [ + ' return applyRule(', + ' maskRanges(', + ' text,', + ' NON_QUERY_SECRET_RULES.flatMap((rule) => ruleRanges(text, rule)),', + ' ),', + ' QUERY_SECRET_RULE,', + ' );', + ].join('\n'), + ['masks a literal query value in the same pass as every other rule'], + 'SECRETS_B4_QUERY_MERGE', + ), + diagnosticControl( + 'diagnostic-mask-merge-extent', + mergeExtent, + ' open.end = range.end;', + ['masks the whole of a range that contains a later one'], + 'SECRETS_B4_MERGE_EXTENT', + ), + diagnosticControl( + 'diagnostic-fixed-point', + llmFixedPoint, + ' return scrubOnce(text);', + ['reaches a fixed point, failing closed when it cannot'], + 'SECRETS_B4_FIXED_POINT', + ), diagnosticControl( 'diagnostic-mask-merge', overlapMerge, @@ -1332,11 +1446,63 @@ export const mutations = [ contactAssignment, String.raw` return userinfo.includes('=', query + 1);`, coreProjectionCases, - 156, + CORE_PROJECTION_CASES, contactHostCase, coreUserinfoContactMarker, rawPasswordAssignments.map((target) => ({ target, marker: coreProjectionMarker })), ), + diagnosticControl( + 'diagnostic-userinfo-authority-rest', + ' return { start, end: start + at + (hostFollows(tail) ? 0 : 1 + tail.length) };', + ' return { start, end: start + at };', + ['masks an ambiguous authority through its end unless a host follows its last @'], + 'SECRETS_B4_AUTHORITY_REST', + ), + diagnosticControl( + 'diagnostic-userinfo-rest-length', + hostFollowsLine, + ' return HOST_SHAPED_AUTHORITY.test(host);', + ['bounds the host test after an ambiguous authority at admitted size'], + 'SECRETS_B4_REST_LENGTH', + ), + coreProjectionControl( + 'core-userinfo-authority-rest', + ' if (!hostFollows(text.slice(maskEnd + 1, restEnd))) maskEnd = restEnd;', + '', + ['masks a secret query value that holds a raw @ after a single-label host'], + 'SECRETS_B4_CORE_AUTHORITY_REST', + ), + coreProjectionControl( + 'core-userinfo-rest-length', + hostFollowsLine, + ' return HOST_SHAPED_AUTHORITY.test(host);', + ['bounds the host test after an ambiguous authority at admitted size'], + 'SECRETS_B4_CORE_REST_LENGTH', + ), + coreProjectionControl( + 'core-scheme-url-lookahead', + coreSchemeCredential, + coreSchemeCredential.replace(String.raw`(?![\w.~+/-]*:\/\/)`, ''), + ['never takes a URL scheme as the token after a scheme label'], + 'SECRETS_B4_CORE_SCHEME_TOKEN', + ), + coreProjectionControl( + 'core-userinfo-split-edges', + ' const urls = redactJoinedCredentialSpans(text, visitUrlUserinfo, undefined, true);', + ' const urls = redactJoinedCredentialSpans(text, visitUrlUserinfo);', + ['\n', '\t'].map( + (separator) => + `masks URL userinfo split at either of its edges (${JSON.stringify(separator)})`, + ), + 'SECRETS_B4_CORE_SPLIT_EDGE', + ), + coreProjectionControl( + 'core-fixed-point', + coreFixedPoint, + ' return redactSecretShapedOnce(text);', + ['reaches a fixed point, failing closed when it cannot'], + 'SECRETS_B4_CORE_FIXED_POINT', + ), // PR95: connection-string schemes join the finite URL list. diagnosticControl( 'diagnostic-connection-scheme-userinfo', From 6b2e94ddf0e94a3e7bb84fcaea61466a3dee11f1 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Tue, 20 Oct 2026 12:53:10 +0300 Subject: [PATCH 14/97] fix(core): address the fifth PR95 review round Round 5 of the PR95 increment's review (at b2c50e88) confirmed 20 findings. Two were high: core's round-4 authority extent erased the context its own later rules read. All are fixed. Core projection (packages/core/src/tools/bounding.ts): - Core's rules run in sequence, so masking through an ambiguous authority's end swallowed the next credential's key name, label, scheme or PEM marker, and in the LF/TAB-joined view it ran across a real line break (SCRUB-R5-1, PR95R5-GEN-1). The extent is gone. An ambiguous userinfo is now masked together with exactly the query values core's opaque-query rule (24+ characters after ?/&) and secret key=value rule would mask whole, ending where those rules end them; the joined view masks the userinfo only. This also closes the tails ruling (2) left after a single-label host (PR95R5-GEN-2) and the nested URL the old resume point skipped (TST-R5-1). - The URL-scheme lookahead after a scheme label is removed: it refused any token whose run reached '://', so tokens glued to a URL stayed visible (SCRUB-R5-2, CONS-R5-1, SEC-R5-1). The split view now masks URL userinfo before scheme labels, as the main pass always did. - A separator inside a URL's 'scheme://' now splits its userinfo, as one at the userinfo's edges does: core had left 'https:/\n/user:pass@host' unmasked since before the increment. - The bounded fixed point moves to a tested helper (TST-R5-6). LLM backstop (packages/llm/src/scrub-secrets.ts): - The split scan retries the next start after a kept candidate whose first letter ends a line, so 'files\nftp://tok\nen@h' (read as sftp) blanks again while a real 'files\nsftp://user@h' stays (SCRUB-R5-3). - The fixed point uses the same tested helper; the CLI's leaf reachability test admits it. Tests and controls: - Discriminating tests for the label rule's trailing ';'/',', its embedded stop's quote run and bound, the nine-label bound's case and LF/TAB letters, an exactly masked password, the escaped joined header name, the split retry, the residual blank after another rule's mask, every core round-5 witness and both fixed-point helpers; each was checked against the mutation it kills (TST-R5-2..5). - The secrets family stays at 110 controls (483 targets and witnesses), the most the unchanged 2,000-file export bound admits: three core controls are replaced for the new mechanisms, two are added (split scheme, split retry) and one pattern-piece control is retired. From this round on, pieces inside a mechanism's pattern are pinned by discriminating tests rather than new controls. Docs: error-handling.md (exactly masked userinfo, core's extent, the residual blank after a mask, the glued-sftp limit's scope), built-in-tools.md, effect-journal.md, the controls README (counts, round 5, the escape-run wording), a PR95 implementation checkpoint in the phase register with status lines in current.md, CLAUDE.md and AGENTS.md, row 9a's split marker, the Step 9 mechanics link and the commit-and-pr skill's development-branch step (DOC-R5-1..4, TST-R5-7, PR95R5-GEN-3, CRIT-R5-1). Refs: ADR-0113 Co-Authored-By: Claude Opus 5.5 --- .claude/skills/commit-and-pr/SKILL.md | 4 +- .github/workflows/ci.yml | 4 +- AGENTS.md | 7 +- CLAUDE.md | 7 +- .../process/supported-leaf-exports.test.ts | 4 +- docs/reference/shared-core/built-in-tools.md | 35 ++-- docs/reference/shared-core/effect-journal.md | 13 +- docs/roadmap/current.md | 6 +- docs/roadmap/deferred-tasks.md | 2 +- .../phase-2.5.5-hardening-and-remediation.md | 41 ++++- docs/standards/error-handling.md | 30 ++-- packages/core/src/tools/bounding.test.ts | 155 ++++++++++++++++-- packages/core/src/tools/bounding.ts | 142 ++++++++++------ packages/core/src/tools/fixed-point.test.ts | 39 +++++ packages/core/src/tools/fixed-point.ts | 13 ++ packages/llm/src/fixed-point.test.ts | 39 +++++ packages/llm/src/fixed-point.ts | 14 ++ packages/llm/src/scrub-secrets.test.ts | 80 ++++++++- packages/llm/src/scrub-secrets.ts | 22 ++- tools/causal-controls/provenance.test.mjs | 18 +- tools/secrets-controls/README.md | 45 +++-- tools/secrets-controls/mutations.mjs | 83 +++++----- 22 files changed, 628 insertions(+), 175 deletions(-) create mode 100644 packages/core/src/tools/fixed-point.test.ts create mode 100644 packages/core/src/tools/fixed-point.ts create mode 100644 packages/llm/src/fixed-point.test.ts create mode 100644 packages/llm/src/fixed-point.ts diff --git a/.claude/skills/commit-and-pr/SKILL.md b/.claude/skills/commit-and-pr/SKILL.md index 69f36dc3..340ae899 100644 --- a/.claude/skills/commit-and-pr/SKILL.md +++ b/.claude/skills/commit-and-pr/SKILL.md @@ -25,10 +25,10 @@ Turn finished work into a clean, auditable commit history and a reviewable PR. R ## Workflow 1. **Gate first.** Run ../standards-check/SKILL.md and `pnpm turbo run lint typecheck test`. Do not commit red. -2. **Branch if on the default branch.** Never commit straight to `main`. +2. **Commit on `development`, never straight to `main`.** Work reaches `main` through a PR from `development`; create no other branch unless the maintainer asks (the [Git workflow](../../../AGENTS.md#git-workflow) in AGENTS.md is the canonical rule). ```bash git rev-parse --abbrev-ref HEAD - # if main: git switch -c feat/- + # if main: git switch development ``` 3. **One scope per commit.** Stage only one package's changes. A change spanning packages is usually two commits; if genuinely atomic, pick the primary scope and name the rest in the body. Do not bundle generated/formatting churn with logic. 4. **Write the message** in Conventional Commits form. Summary is imperative, lower-case, no trailing period, ≤ ~72 chars. Body (expected for anything non-obvious) explains the *why*, not a restatement of the diff. Reference the ADR in a trailer; end with the required co-author trailer. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e375554..6b4131bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -281,10 +281,10 @@ jobs: secrets-mutations: name: Wave 2 secrets causal controls # The historical 35-control, three-phase capture exceeded the former aggregate budget. - # Wave 3's PR95 increment grows the cohort to 109 controls (2026-10-20): each scrubber + # Wave 3's PR95 increment grows the cohort to 110 controls (2026-10-20): each scrubber # control runs its eight-MiB-heavy file in three phases (about 24 s locally), so the job # allowance matches the filesystem job's 60 minutes. Per-phase, case/hook, output and - # retained-evidence bounds remain unchanged (109 × 18 + 7 = 1,969 of 2,000 files); + # retained-evidence bounds remain unchanged (110 × 18 + 7 = 1,987 of 2,000 files); # this comment does not certify execution. timeout-minutes: 60 runs-on: ubuntu-latest diff --git a/AGENTS.md b/AGENTS.md index 161cf60b..42b55d97 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -124,9 +124,10 @@ only and was then independently reviewed (four rounds and a security sitting, wi fixes on `development`, not yet on `main`); CI for it and the merge commit finished green, and SonarCloud's `main` gate (red since at least PR #90) was cleared on 2026-10-19. A pre-implementation review of Steps 9–19 and the maintainer's [2026-10-19 rulings](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) -give every open PR #95 finding an owner and record 29 increments (no new ADR). Next is the PR95 -increment (the scrubber residuals, including both security residuals), then Step 9a.1 (release -map/custody substrate, #46/ADR-0111). `development` is pushed after each accepted increment. No provider, +give every open PR #95 finding an owner and record 29 increments (no new ADR). The PR95 increment +(the scrubber residuals, including both security residuals) is implemented on `development` and under +review: rounds 1–5 were NONCLEAN and are corrected ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)), and acceptance +needs two consecutive clean rounds. Then Step 9a.1 (release map/custody substrate, #46/ADR-0111). `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. See [docs/roadmap/current.md](docs/roadmap/current.md) for live status. diff --git a/CLAUDE.md b/CLAUDE.md index b8350fb4..b4acc257 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -213,9 +213,10 @@ only and was then independently reviewed (four rounds and a security sitting, wi fixes on `development`, not yet on `main`); CI for it and the merge commit finished green, and SonarCloud's `main` gate (red since at least PR #90) was cleared on 2026-10-19. A pre-implementation review of Steps 9–19 and the maintainer's [2026-10-19 rulings](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) -give every open PR #95 finding an owner and record 29 increments (no new ADR). Next is the PR95 -increment (the scrubber residuals, including both security residuals), then Step 9a.1 (release -map/custody substrate, #46/ADR-0111). `development` is pushed after each accepted increment. No provider, +give every open PR #95 finding an owner and record 29 increments (no new ADR). The PR95 increment +(the scrubber residuals, including both security residuals) is implemented on `development` and under +review: rounds 1–5 were NONCLEAN and are corrected ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)), and acceptance +needs two consecutive clean rounds. Then Step 9a.1 (release map/custody substrate, #46/ADR-0111). `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. For live status, per-PR history, milestone dates, and open obligations, see the canonical diff --git a/apps/cli/src/process/supported-leaf-exports.test.ts b/apps/cli/src/process/supported-leaf-exports.test.ts index d50e5336..c7518d32 100644 --- a/apps/cli/src/process/supported-leaf-exports.test.ts +++ b/apps/cli/src/process/supported-leaf-exports.test.ts @@ -75,8 +75,10 @@ describe('supported compiled dependency leaves', () => { const leaf = pathToFileURL(require.resolve('@relavium/llm/scrub-secrets')).href; const allowed = [ leaf, - // The leaf's dependency-free, package-internal split-credential probe (no package export). + // The leaf's dependency-free, package-internal split-credential probe and fixed-point + // helper (no package exports). new URL('./split-credential-probe.js', leaf).href, + new URL('./fixed-point.js', leaf).href, pathToFileURL(require.resolve('@relavium/shared/terminal-safe')).href, ]; // An isolated ESM loader refuses every other module, including schema/provider barrels. diff --git a/docs/reference/shared-core/built-in-tools.md b/docs/reference/shared-core/built-in-tools.md index e386b4f6..a1ab0576 100644 --- a/docs/reference/shared-core/built-in-tools.md +++ b/docs/reference/shared-core/built-in-tools.md @@ -96,25 +96,32 @@ existing content-sensitive handling. A raw `?` or `#` before a URL authority's `@` begins public query or fragment text only when the text before it is host-shaped (a dotted name or IPv4, `localhost` or a bracketed IP literal, with an optional port) **and** an `=` assignment follows it; otherwise the userinfo is masked. -Masked userinfo that holds such a raw `?` or `#` leaves the authority ambiguous: unless a -host-shaped name follows its last `@`, the text there may be the rest of a query value -(`https://svc?password=P@ss`), so the mask runs through the authority's end, the next `/`, -whitespace, quote, `,` or `}`. This is the same predicate and extent as the +This is the same predicate as the [LLM diagnostic backstop](../../standards/error-handling.md#diagnostic-secret-backstop), including -its 260-character bound on a host-shaped name. The projection otherwise differs from -that backstop by design: its authority also ends at `"`, `'`, `,`, `}` and `]`, so userinfo that -contains one of them — an already-masked `[REDACTED]` password among them — is left as it is; it -recognizes any scheme of up to 32 characters rather than a finite list; and a userinfo with no -password is kept as a vanity handle below 16 characters and masked from 16 on, under every scheme. +its 260-character bound on the host-shaped prefix. Masked userinfo that holds such a raw `?` or `#` +leaves the authority ambiguous: the userinfo may also be the query a later rule reads +(`https://svc?password=P@ss`). The backstop merges every rule's ranges and masks through the +authority's end; this projection's rules run in sequence, and masking the userinfo first would +remove the names its later rules read, so the userinfo mask also covers each query value that its +opaque-query rule (24 or more characters after `?`/`&`) or secret key/value rule would mask whole, +and ends where that rule would end it. It never masks farther, so the context of a credential after +the authority stays in place; in the LF/TAB-joined view it masks the userinfo only. The projection +otherwise differs from that backstop by design: its authority also ends at `"`, `'`, `,`, `}` and +`]`, so userinfo that contains one of them — an already-masked `[REDACTED]` password among them — +is left as it is; it recognizes any scheme of up to 32 characters rather than a finite list; and a +userinfo with no password is kept as a vanity handle below 16 characters and masked from 16 on, +under every scheme. LF/TAB inside a recognized token-shaped credential extends the masked original span. An ambiguous token-alphabet continuation after LF/TAB is conservatively masked with that credential. An LF or TAB right after a URL's `://` or right before its userinfo's `@` -belongs to the split userinfo, which is masked as one span. An authentication scheme label -never takes a URL scheme as its token: `token postgres://app:pw@db` leaves the URL to the -userinfo rule. The projection repeats until its output stops changing, at most three passes, -and becomes `[redacted]` if it has not settled, so projecting a projection changes nothing. Ordinary spaces and punctuation separate public prose; the masker -does not discard the whole diagnostic because a credential appears. Supported split +belongs to the split userinfo, which is masked as one span. URL userinfo is masked before an +authentication scheme label takes its token, in the split view as in the main pass, so a label +that takes a URL scheme (`token postgres://app:pw@db` → `token [redacted]://[redacted]@db`) never +hides a password. The projection repeats until its output stops changing, at most three passes, +and becomes `[redacted]` if it has not settled, so projecting a projection changes nothing. +Ordinary spaces and punctuation separate public prose; the masker does not discard the whole +diagnostic because a credential appears. Supported split shapes are recognized **before** complete-text one-line normalization, followed by ordinary recognition and length clipping. This preserves multiline quoted-value recognition without exposing a split token's suffix. Public argument bytes are not diff --git a/docs/reference/shared-core/effect-journal.md b/docs/reference/shared-core/effect-journal.md index 95c2a18b..ee50c291 100644 --- a/docs/reference/shared-core/effect-journal.md +++ b/docs/reference/shared-core/effect-journal.md @@ -370,8 +370,8 @@ inline-media redaction remain deliberate projection equality; raw credential byt added to the digest to distinguish calls. Recognized URL userinfo is masked through the authority's last `@`, preserving the host, -path, query and fragment; userinfo holding a raw `?` or `#` with no host-shaped name after -that `@` is masked through the authority's end. Recognized opaque URL-query values are masked through their delimiter, including values +path, query and fragment; userinfo holding a raw `?` or `#` is masked together with the query +values the later rules would mask whole. Recognized opaque URL-query values are masked through their delimiter, including values longer than 512 characters. Quoted secret-key values honor escaped quotes and backslashes through the actual closing quote. A recognized value must not leave a credential suffix in the projection. This remains a finite shape detector, not a claim to identify arbitrary @@ -380,10 +380,11 @@ unlabelled secrets. Existing old digests are not retroactively erased or rewritt A raw `?` or `#` before the authority's `@` is public only after a host-shaped prefix that an `=` assignment follows ([built-in tools](built-in-tools.md#finite-diagnostic-credential-masking)). Wave 3's PR95 increment tightened that predicate without a new digest domain. It also masks -an ambiguous authority through its end, masks userinfo split by an LF or TAB at either edge, -keeps an authentication scheme label from taking a URL scheme as its token, and repeats the -projection until it is stable. These change a `v5:` projection only for the arguments they -project differently, such as userinfo previously treated as public: a resumed run whose +an ambiguous userinfo together with the query values the later rules would mask whole, masks +userinfo split by an LF or TAB at either edge, masks URL userinfo before an authentication +scheme label in the split view, and repeats the projection until it is stable. These change a +`v5:` projection only for the arguments they project differently, such as userinfo previously +treated as public: a resumed run whose committed effect carried one now computes a different digest and refuses matching prepare instead of replaying, which is the fail-closed outcome of any digest difference. Every other `v5:` row keeps its digest. diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index 4bb4ba4f..f06a8b43 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -164,8 +164,10 @@ security residuals that predate the follow-up — an `api-key:Bearer ` to Wave 3 implementation continues on `development`: Steps 1–8 are merged. A pre-implementation document/source review of Steps 9–19 and the maintainer's rulings are recorded in the [2026-10-19 rulings](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) -(no new ADR; 29 recorded increments). **Next is the PR95 increment** — the PR #95 scrubber residuals, including the two security -residuals — and then **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), whose +(no new ADR; 29 recorded increments). **The PR95 increment is implemented on `development` and under review** — the PR #95 +scrubber residuals, including the two security residuals; rounds 1–5 were NONCLEAN and are corrected, as its +[implementation checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20) records, and acceptance needs two consecutive clean rounds after the last +material fix. **Step 9a.1** follows, the first increment of the release map/custody substrate (#46, ADR-0111), whose 9a.1–9d increments carry a dedicated archive/crypto security sitting. Steps 10–19 follow the [increment table](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) and end with whole-wave acceptance at Step 19. `development` is pushed after each accepted increment; PR #95 no longer bounds the diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index c8ffad93..4a60efce 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -2252,7 +2252,7 @@ future test cannot silently re-acquire it. ## Wave 3 approved later-wave ownership — 2026-10-16 - [ ] **Scheduled → Wave 4b: production recipient key constraints.** Approved with the Wave 3 - Steps 9–19 rulings (2026-10-19, [ruling C2](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19)): + Steps 9–19 rulings (2026-10-19, [Step 9 mechanics ruling](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19)): the confirmed production recipient for encrypted release custody must be an OpenPGP v4 key with a cv25519 or RSA≥3072 encryption-capable subkey. Wave 3 Step 9 implements custody with synthetic keys only; an actual release fails closed until this recipient, its offline diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index b76a25d0..4ad5fd68 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -3696,7 +3696,7 @@ No credential values or request captures belong in tracked evidence. | 8d | **Recorded split — sessions, terminal and activation:** fresh/resumed/one-shot/Home/pending/replaced session lifetimes, retained publication failure and shared actual cleanup promises; connect caught slash/rebuild faults, restore terminal custody before diagnosis, then activate early coordinator/background/sink integration. Actual native/platform/packed controls and complete cumulative Step 8 acceptance follow. | | PR95 | **Recorded plan amendment, 2026-10-19 (maintainer-approved; new scope from the [PR #95 independent review](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18), not a split of rows 1–19) — scrubber residuals, before 9a:** the `api-key:Bearer`/`Basic` label erasure that leaks the token; raw `?`/`#` URL userinfo masked unless the text before it is host-shaped **and** an `=` assignment separates it from the authority `@`, applied to the LLM leaf and core's twin predicate; an extended finite scheme list (postgres, postgresql, mysql, mariadb, redis, rediss, mongodb, mongodb+srv, amqp, amqps, ssh, sftp, ldap, ldaps, smtp, smtps) masking colon-bearing userinfo for the new schemes; module-load checks of the split-credential scanner's two invariants, a start prefilter, joined-view and glued-scheme witnesses, the load-sensitive 8 MiB timing case, and the causal-control pins. One increment with ordered commits, two NEW clean complete cumulative rounds and a separate security sitting. | | 9 | **Release map/custody substrate:** #46/ADR-0111, independently reviewed after the finite graph. Upstream TS maps, sole prepack final build, pre-hash trailer removal/staging, exact tarball/CLI+engine symbolication, encrypt-once candidate with permissioned prebuild discovery/reservation → pack/reuse → smoke → archive → npm → finalization. Required artifact/platform aggregation, all rerun modes, lost-staging refusal and permission-enforcing offline API controls. Dedicated archive/crypto security sitting. Fixture proof never claims production recipient/custody configuration. | -| 9a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage #46/ADR-0111 unchanged, no new contract) — release maps, final build and symbolication:** fresh uncached upstream original-TypeScript maps, the sole prepack final CLI build, final trailer removal before hashes, map and private-graph staging outside public artifacts and caches, exact tarball-to-final-JS binding, and real packed CLI and engine frame symbolication with wrong JS/map/source and missing-map negative controls. Ordinary builds remain map-free. | +| 9a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage #46/ADR-0111 unchanged, no new contract) — release maps, final build and symbolication:** fresh uncached upstream original-TypeScript maps, the sole prepack final CLI build, final trailer removal before hashes, map and private-graph staging outside public artifacts and caches, exact tarball-to-final-JS binding, and real packed CLI and engine frame symbolication with wrong JS/map/source and missing-map negative controls. Ordinary builds remain map-free. *Split again on 2026-10-19 into 9a.1 and 9a.2 below, which replace it as increments ([Plan shape](#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19)).* | | 9a.1 | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — candidate build and finalizer:** an owned isolated candidate copy, fresh uncached upstream maps, candidate-mode tsup selection with absent `sourcesContent`, the prepack finalizer (upstream and final map validation, repository-relative sources, exact trailer removal before hashes), owner-only private staging outside public artifacts and caches, closure/seal adaptation, the ordinary map-free control, the PR #95 archive/worker fold-ins, and a fail-closed `release.yml` publish guard that refuses any non-release candidate until 9d and Wave 4b custody exist. The receipt stays `compiled_package_test`. | | 9a.2 | **Recorded split — manifest, symbolication and package-test receipt:** the private map manifest keyed by `dist/.js`, actual packed CLI and engine frame symbolication against the matching source revision, wrong JS/map/source and missing/malformed-upstream negative controls (product copies via `check:map-controls`, also a required uncached CI job), and the atomic `package_test` receipt flip across seal, binding, consumers, aggregate and both workflows. | | 9b | **Recorded split — recipient custody:** synthetic-only isolated GnuPG controls (public-only home/keyring, pinned primary and encryption-subkey fingerprints, machine-readable refusal of missing, expired, revoked or unusable recipients), encrypt once and reuse the original ciphertext. Explicit crypto/custody security sitting. Fixture proof never claims production recipient or custody configuration. | @@ -3736,7 +3736,7 @@ No credential values or request captures belong in tracked evidence. | 19b | **Recorded split — whole-wave acceptance:** every disposition, canonical landings and the Proposed-section inventory, final exact-head gates, two NEW whole-wave cumulative reviews and the PR. | Steps are dependency-safe increments, not permission to parallel-edit shared hot -files. Within each row, arrows remain strict. The process/packaging sequence replaces the old fatal-before-lazy assumption: pure contracts → guarded lazy graph → native fatal integration → map/archive custody → remaining CLI behaviour. It is nineteen increments, with no parallel edits of shared hot files. Step 1 completes before all other +files. Within each row, arrows remain strict. The process/packaging sequence replaces the old fatal-before-lazy assumption: pure contracts → guarded lazy graph → native fatal integration → map/archive custody → remaining CLI behaviour. It was approved as nineteen steps; the recorded splits and the 2026-10-19 amendment make it 29 increments (see the rulings' Plan shape), with no parallel edits of shared hot files. Step 1 completes before all other code. Home cost is disposed or implemented before its decomposition; viewport geometry work never races editor/cache work. If review exposes a necessary smaller increment, split it with unchanged ID coverage and record the split before proceeding. @@ -9415,3 +9415,40 @@ cause-wrap and the purity-guard root set → 10a; `HomeDeps.exit` → 14a; the k exit-code literal sites → 17a; test-evidence and join-table nits → 17b; the duplicate-heading nit (with its corrected premise) → 19b. [deferred-tasks.md](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18) carries the same owners. + +### Wave 3 PR95 increment implementation checkpoint and review-round corrections — 2026-10-20 + +**Scope and source.** The PR95 increment implements the PR95 scrubber residuals ruling above in +`packages/llm/src/scrub-secrets.ts` (with its package-internal `split-credential-probe.ts` and +`fixed-point.ts`) and in core's twin projection, `packages/core/src/tools/bounding.ts`, together with +the CLI's authored-content detector, the secrets causal controls and the canonical documents. Base +`9902e0a0`; commits `38e10d42` (Docs-0, the rulings), `dda3b45e` (implementation) and the +corrections `ef3ded0d`, `bdd292a6`, `6fb7e9d9`, `b2c50e88` and the round-5 commit that adds this +section. The commits stay local on `development` until the increment is accepted. + +**Review rounds.** Each round is a NEW independent complete cumulative review of `9902e0a0..HEAD`, every +reviewer in its own worktree detached at the reviewed head: scrubber correctness, tests and causal +controls, documentation, integration, a separate security sitting and, from round 3, generative +properties and a completeness critic. Every finding is adversarially verified. The detailed reports +stay local and Git-ignored. + +| Round | Reviewed head | Verdict | Verified findings | Main corrections | +| --- | --- | --- | --- | --- | +| 1 | `dda3b45e` | NONCLEAN | 15 (2 high) | the host-shaped prefix bounded at 260 characters before the RegExp, which had exhausted V8's stack at admitted size, and anchored; stacked labels; already-masked userinfo kept | +| 2 | `ef3ded0d` | NONCLEAN | 15 | quote-closed labels, bounded label stacking, masked tokens as fixed points | +| 3 | `bdd292a6` | NONCLEAN | 17 | sequential rule application replaced by one merged, token-only masking pass | +| 4 | `6fb7e9d9` | NONCLEAN (integration clean) | 19 | the literal query names in the merged pass; concatenated label runs and the nine-label bound; one standalone label rule for both labels; a bounded fixed point in both maskers; the secrets CI job allowance becomes 60 minutes | +| 5 | `b2c50e88` | NONCLEAN | 20 (2 high) | core's round-4 authority extent erased the context its sequential rules read, so core now masks an ambiguous userinfo with exactly the query values its later rules mask whole; its split view masks URL userinfo before scheme labels; the LLM split scan retries a kept candidate glued across a line break; tested fixed-point helpers; untested pattern pieces gained discriminating tests | + +**Author gates and controls.** Every corrected head passed `pnpm run ci` and the affected suites under +`CI=true GITHUB_ACTIONS=true`, plus local differential, exhaustive and admitted-size harnesses that +found no new leak against the previous head. The secrets family is 110 controls / 330 phases / 482 +targets and witnesses, projecting 1,987 of the unchanged 2,000 retained files; +[tools/secrets-controls/README.md](../../../tools/secrets-controls/README.md) records each round's +control changes and, from round 5, the rule that pieces inside a mechanism's pattern are pinned by +discriminating tests rather than by new controls. + +**Remaining gates.** The increment is not independently accepted: two NEW consecutive clean complete +cumulative rounds after the last material fix, with a clean separate security sitting, remain. Then +`development` is pushed and its CI observed, and the PR #95 items in +[deferred-tasks.md](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18) close. diff --git a/docs/standards/error-handling.md b/docs/standards/error-handling.md index 40f80fa7..989deff9 100644 --- a/docs/standards/error-handling.md +++ b/docs/standards/error-handling.md @@ -164,11 +164,13 @@ content. `postgres`, `postgresql`, `mysql`, `mariadb`, `redis`, `rediss`, `mongodb`, `mongodb+srv`, `amqp`, `amqps`, `ssh`, `sftp`, `ldap`, `ldaps`, `smtp` and `smtps` mask only colon-bearing userinfo (`user:pass@`, `:pass@`), so a bare user such as `ssh://git@host` (or a colonless - `redis://#TOKEN@host`) stays visible. Userinfo that is already masked stays as it is: an - exactly masked password under a connection-string scheme, or every part masked or empty under - any scheme. The authored-content detector's projection of `postgres://app:{{secrets.password}}@db` - to `postgres://app:[REDACTED]@db` is therefore a fixed point. Core's - journal projection applies the same `?`/`#` predicate and ambiguous-authority extent + `redis://#TOKEN@host`) stays visible. Userinfo whose password is exactly masked stays as it is: + beside any username under a connection-string scheme, or beside a masked or empty username under + any scheme; an empty password, or text glued after a masked one, is still masked. The + authored-content detector's projection of `postgres://app:{{secrets.password}}@db` to + `postgres://app:[REDACTED]@db` is therefore a fixed point. Core's journal projection applies the + same `?`/`#` predicate, but its rules run in sequence, so it masks an ambiguous userinfo together + with the query values its later rules would mask whole instead of through the authority's end ([built-in tools](../reference/shared-core/built-in-tools.md#finite-diagnostic-credential-masking)). - **Split and normalized forms.** Every rule except the literal query names also probes the terminal-normalized text; a credential revealed only by normalization masks the **whole** @@ -179,7 +181,10 @@ content. - **Documented trade-offs, all failing closed.** A scheme glued to a word character is masked in place; an unlisted scheme ending in a listed one (`gftp://`, `jdbc:mysql://`) matches by its suffix; prose that the whitespace-stripped residual probe glues to a path-less URL and a later - `@` blanks the whole diagnostic; a host fragment with no assignment (`https://host.example#sec@x`), + `@` blanks the whole diagnostic, and so does a key prefix (`sk-`, `AIza`) that the same probe + joins with the following words to its minimum length, at the start of the text or, since the + scrub repeats, right after another rule's mask (`Authorization: Bearer xyz sk-ant api key + invalid`); a host fragment with no assignment (`https://host.example#sec@x`), a single-label host query and an over-long host-shaped prefix are masked, and so is the text after the last `@` of such an ambiguous authority unless it is host-shaped; a `Bearer`/`Basic` label that ends a line takes the next token with it; an opaque value after a @@ -197,12 +202,13 @@ content. fragment after such a break stays visible, as it does after an ordinary space; a label glued to a quoted token without whitespace (`Bearer""`) is likewise outside the shape, as is a token after a JSON escape (`\n`, `\t`) that stands for the whitespace after a label. A word - glued before `sftp://` (`statusftp://user@h`) reads as `sftp`, so its colonless userinfo stays - visible like any `sftp` user: every whitespace-joined probe, the CLI's display residual among - them, glues prose to a scheme the same way, and reading it as `ftp` would blank an ordinary - `connect to sftp://user@host` diagnostic. Malformed URLs (`https:/u:p@h`), a raw `/` inside a - password, invisible characters inside a scheme and JSON whose quotes are themselves escaped as - `\u0022` are outside the finite shapes. + glued before `sftp://` on the same line (`statusftp://user@h`) reads as `sftp`, so its colonless + userinfo stays visible like any `sftp` user: every whitespace-joined probe, the CLI's display + residual among them, glues prose to a scheme the same way, and reading it as `ftp` would blank + an ordinary `connect to sftp://user@host` diagnostic. (A word that ends a line before `ftp://` + is no such glue: an LF/TAB-split ftp credential after it blanks the diagnostic.) Malformed URLs + (`https:/u:p@h`), a raw `/` inside a password, invisible characters inside a scheme and JSON + whose quotes are themselves escaped as `\u0022` are outside the finite shapes. ## No silent catches diff --git a/packages/core/src/tools/bounding.test.ts b/packages/core/src/tools/bounding.test.ts index 9d25a034..897ea9ab 100644 --- a/packages/core/src/tools/bounding.test.ts +++ b/packages/core/src/tools/bounding.test.ts @@ -1289,30 +1289,37 @@ describe('PR95 review round 4: ambiguous authorities, URL schemes after labels a } }); - it('bounds the host test after an ambiguous authority at admitted size', () => { + it('masks an admitted-size query value inside an ambiguous userinfo', () => { const labels = 'a.'.repeat(Math.floor((8 * 1024 * 1024 - 256 - 64) / 2)); const input = `upstream said https://user:pw?x=1@${labels}a failed`; let output = ''; expect(() => { output = redactSecretShapedText(input); - }, 'SECRETS_B4_CORE_REST_LENGTH').not.toThrow(); + }, 'SECRETS_B5_CORE_QUERY_EXTENT').not.toThrow(); expect( output === 'upstream said https://[redacted] failed', - 'SECRETS_B4_CORE_REST_LENGTH', + 'SECRETS_B5_CORE_QUERY_EXTENT', ).toBe(true); }, 15_000); - it('never takes a URL scheme as the token after a scheme label', () => { + it('masks the userinfo of a URL behind a scheme label, split or not, and a token glued to a URL', () => { for (const [input, expected] of [ [ 'invalid token postgres://app:hunter2secret@db\n', - 'invalid token postgres://[redacted]@db\n', + 'invalid token [redacted]://[redacted]@db\n', ], - ['invalid bearer https://app:hunter2secret@db\n', 'invalid bearer https://[redacted]@db\n'], - ['token postgresql://app:hunter2secret@db', 'token postgresql://[redacted]@db'], + ['invalid token postgres://app:pa#ss@db\n', 'invalid token [redacted]://[redacted]@db\n'], + ['token postgresql://app:hunter2secret@db', 'token [redacted]://[redacted]@db'], ['token abcdefghijkl\n', 'token [redacted]\n'], + ['Bearer abcdefgh12345678/https://x.example', 'Bearer [redacted]://x.example'], + ['Bearer abcDEF123456xyzhttps://h.example', 'Bearer [redacted]://h.example'], + ['token\tabcdefgh12345678/https://x', 'token [redacted]://x'], + [ + 'Authorization: Bearer abcDEF123456xyzhttps://u:p@h.example', + '[redacted] [redacted]://[redacted]@h.example', + ], ] as const) { - expect(redactSecretShapedText(input), 'SECRETS_B4_CORE_SCHEME_TOKEN').toBe(expected); + expect(redactSecretShapedText(input), 'SECRETS_B5_CORE_SCHEME_ORDER').toBe(expected); } }); @@ -1327,9 +1334,10 @@ describe('PR95 review round 4: ambiguous authorities, URL schemes after labels a } }); - it('reaches a fixed point, failing closed when it cannot', () => { + it('reaches a fixed point', () => { + // The opaque value removes the authority's last `@`, so the second pass reads a new userinfo. for (const [input, expected] of [ - ['g://s.hos#mHPGEAXXQ8@?q==a@b.exam/user:3eYPUzfxo', 'g://[redacted]'], + ['g://s.hos#mHPGEAXXQ8@?q==a@b.exam/user:3eYPUzfxo', 'g://[redacted]@?q=[redacted]'], ['[redacted]', '[redacted]'], ] as const) { const output = redactSecretShapedText(input); @@ -1338,3 +1346,130 @@ describe('PR95 review round 4: ambiguous authorities, URL schemes after labels a } }); }); + +describe('PR95 review round 5: an ambiguous userinfo masked with exactly the values later rules read', () => { + const pem = [ + 'https://a?b:c@-----BEGIN RSA PRIVATE KEY-----', + 'MIIBOgIBAAJBAKj34GkxFhD90vcNLYLInFEX6Ppy1tPf9Cnzj4p4WGeKLs1Pt8Qu', + '-----END RSA PRIVATE KEY-----', + ].join('\n'); + const projects = (marker: string, pairs: readonly (readonly [string, string, string])[]) => { + for (const [input, expected, secret] of pairs) { + const output = redactSecretShapedText(input); + expect(output, marker).toBe(expected); + expect(output.includes(secret), marker).toBe(false); + expect(redactSecretShapedText(output), marker).toBe(output); + } + }; + + it('masks an ambiguous userinfo with the query values its later rules would mask whole', () => { + projects('SECRETS_B5_CORE_QUERY_EXTENT', [ + ['mysql://db?user=root&password=Adm1n@2024.prod', 'mysql://[redacted]', '2024.prod'], + ['https://gateway?password=P@ss.word1', 'https://[redacted]', 'ss.word1'], + ['https://svc?client_secret=abcdefgh@def.example', 'https://[redacted]', 'def.example'], + ['https://svc?x.client_secret=abcdefgh@def.example', 'https://[redacted]', 'def.example'], + [ + 'postgres://db?sslmode=require&password=Adm1n@x/Secret2024', + 'postgres://[redacted]', + 'Secret2024', + ], + [ + 'see https://svc?token=abcdefghij@foo.example/x for details', + 'see https://[redacted] for details', + 'foo.example', + ], + ['https://user:pw?token=abcdefgh@host', 'https://[redacted]', 'host'], + [`https://svc?sig=${'A'.repeat(24)}@tail.example`, 'https://[redacted]', 'tail.example'], + [ + 'https://proxy?redirect_target=a@http://user:secretpw@host.example/', + 'https://[redacted]', + 'secretpw', + ], + ['https://svc:pw?x=1@ftp://user:pw2secret@h.example/', 'https://[redacted]', 'pw2secret'], + ]); + // A value no later rule masks leaves the userinfo mask where it was. + for (const [input, expected] of [ + ['https://us?er=1:TOKEN@h.example/', 'https://[redacted]@h.example/'], + ['https://alice:pa?ss@provider.invalid/v1', 'https://[redacted]@provider.invalid/v1'], + ] as const) { + expect(redactSecretShapedText(input), 'SECRETS_B5_CORE_QUERY_EXTENT').toBe(expected); + } + }); + + it('keeps the context of a later credential after an ambiguous authority', () => { + projects('SECRETS_B5_CORE_LATER_CONTEXT', [ + [ + 'url: postgres://app:pa#ss@db.internal\npassword: hunter2hunter2\n', + 'url: postgres://[redacted]@db.internal\n[redacted]\n', + 'hunter2hunter2', + ], + [ + 'url: postgres://app:pa#ss@db.internal\n\ttoken: hunter2hunter2\n', + 'url: postgres://[redacted]@db.internal\n\t[redacted]\n', + 'hunter2hunter2', + ], + [ + 'https://user:pa?ss@minio?sig=Zm9vYmFy/YmF6cXV4cXV1eHh4eHh4eHh4eHh4', + 'https://[redacted]@minio?sig=[redacted]', + 'YmF6cXV4', + ], + [ + 'https://a?b:c@Bearer abcdefgh12345678', + 'https://[redacted]@Bearer [redacted]', + 'abcdefgh12345678', + ], + [pem, 'https://[redacted]@[redacted]', 'MIIBOgIBAAJBAKj34'], + [ + 'DATABASE_URL=mysql://db?user=root&password=Adm1n@2024\nREPLICA_URL=postgres://app:hunter2secret@replica/x', + 'DATABASE_URL=mysql://[redacted]\nREPLICA_URL=postgres://[redacted]@replica/x', + 'hunter2secret', + ], + [ + 'https://gateway?password=P@ssw0rd123\nBearer hunter2secretvalue', + 'https://[redacted]\nBearer [redacted]', + 'hunter2secretvalue', + ], + [ + 'https://gateway?password=P@ssw0rd123\tx-api-key: hunter2secretvalue', + 'https://[redacted]\t[redacted]', + 'hunter2secretvalue', + ], + [ + 'mysql://db?user=root&password=Adm1n@2024;Bearer hunter2secretvalue', + 'mysql://[redacted];Bearer [redacted]', + 'hunter2secretvalue', + ], + ]); + }); + + it.each(['\n', '\t'])( + 'masks URL userinfo whose scheme is split by an LF or TAB (%j)', + (separator) => { + for (const [input, expected] of [ + [ + `https:/${separator}/user:hunter2secret@host.example/x`, + `https:/${separator}/[redacted]@host.example/x`, + ], + [ + `https${separator}://user:hunter2secret@host.example`, + `https${separator}://[redacted]@host.example`, + ], + [ + `ssh:${separator}//deploy:hunter2secret@h.example`, + `ssh:${separator}//[redacted]@h.example`, + ], + ] as const) { + expect(redactSecretShapedText(input), 'SECRETS_B5_CORE_SPLIT_SCHEME').toBe(expected); + } + }, + ); + + it('masks a token glued to a URL at admitted size', () => { + const input = `Bearer ${'a'.repeat(8 * 1024 * 1024 - 64)}://h.example`; + let output = ''; + expect(() => { + output = redactSecretShapedText(input); + }, 'SECRETS_B5_CORE_SCHEME_ORDER').not.toThrow(); + expect(output === 'Bearer [redacted]://h.example', 'SECRETS_B5_CORE_SCHEME_ORDER').toBe(true); + }, 15_000); +}); diff --git a/packages/core/src/tools/bounding.ts b/packages/core/src/tools/bounding.ts index 27619e9e..f6e8d2fd 100644 --- a/packages/core/src/tools/bounding.ts +++ b/packages/core/src/tools/bounding.ts @@ -23,6 +23,7 @@ import { utf8ByteLength } from '@relavium/shared'; // while the IMPLEMENTATION lives in `@relavium/shared` — see its header for why it moved. export { utf8ByteLength }; +import { toFixedPoint } from './fixed-point.js'; import type { ToolHost, ToolResultLimits } from './types.js'; export interface BoundedResult { @@ -159,11 +160,38 @@ const STANDALONE_CREDENTIAL_PREFIX = new RegExp( // a legitimate boundary or manufacture one inside a public word. The source uses the same fragments. const JOINED_STANDALONE_CREDENTIAL = new RegExp(STANDALONE_CREDENTIAL.source.slice(2), 'y'); -// A token is never a URL scheme: `token postgres://app:pw@db` leaves the URL to the userinfo rule. const SCHEME_CREDENTIAL = - /\b(b[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|b[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c|t[\n\t]*o[\n\t]*k[\n\t]*e[\n\t]*n)\s+(?=[\w.~+/-]{8})(?![\w.~+/-]*:\/\/)[\w.~+/-]+={0,2}/gi; + /\b(b[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|b[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c|t[\n\t]*o[\n\t]*k[\n\t]*e[\n\t]*n)\s+(?=[\w.~+/-]{8})[\w.~+/-]+={0,2}/gi; + +// A secret-ish key (bounded wrappers keep this ReDoS-safe) + `=`/`:` + its value. The optional `["']?` +// BEFORE the separator catches the JSON `"access_token":"…"` shape (an OAuth/egress response body). The +// value has QUOTED and UNQUOTED branches. A quoted value ends at the first UNESCAPED matching quote, +// including JSON's escaped quotes/backslashes; otherwise a valid password such as `a\"secret-tail` +// left its tail exposed. The two alternatives inside each repetition are disjoint: a backslash +// consumes exactly the following non-newline character, while the ordinary class excludes it. +// No ambiguous nested repetition/backref is needed. Unquoted values retain their delimiter and +// minimum-length policy. Kept a single analyzable LITERAL (not composed via `new RegExp`) so Sonar's static +// super-linear-runtime (S5852) check still covers this security-critical pattern; its keyword-alternation +// breadth is deliberate (and `apikey` is dropped — `api[_-]?key` already subsumes it). +const SECRET_KEY_VALUE = + /\b[\w-]{0,32}(?:password|passwd|secret|token|api[_-]?key|authorization|access[_-]?key|private[_-]?key|client[_-]?secret)[\w-]{0,16}["']?\s*[=:]\s*(?:"(?:[^"\\\r\n]|\\[^\r\n])*"|'(?:[^'\\\r\n]|\\[^\r\n])*'|(?=[^\s"',;&]{6})[^\s"',;&]+)/gi; +/** The same rule anchored at one position, for a query value inside an ambiguous userinfo. */ +const SECRET_KEY_VALUE_AT = new RegExp(SECRET_KEY_VALUE.source, 'iy'); +/** A query name the secret key=value rule reads, from the word boundary where that rule would start. */ +const SECRET_QUERY_NAME = + /\b[\w-]{0,32}(?:password|passwd|secret|token|api[_-]?key|authorization|access[_-]?key|private[_-]?key|client[_-]?secret)[\w-]{0,16}$/i; -type SecretSpanVisitor = (start: number, end: number, replacement: string) => void; +/** + * Masks one recognized span. `splitFrom`, when given, is where the credential's identifying prefix + * starts (a URL's scheme): a removed LF/TAB anywhere from there on splits the credential, so a + * separator inside `scheme://` counts even though only the userinfo is replaced. + */ +type SecretSpanVisitor = ( + start: number, + end: number, + replacement: string, + splitFrom?: number, +) => void; /** * A failed JWT probe must not rescan an admitted-size token run at every original boundary, @@ -320,11 +348,12 @@ function redactJoinedCredentialSpans( } return target + removed; } - visit(joined, (start, end, replacement) => { + visit(joined, (start, end, replacement, splitFrom) => { + const originalSplitFrom = splitFrom === undefined ? undefined : advance(splitFrom); const originalStart = advance(start, !ownsEdges); const originalEnd = advance(end, ownsEdges); if (boundary?.test(text[originalStart - 1] ?? '')) return; - if (!/[\n\t]/.test(text.slice(originalStart, originalEnd))) return; + if (!/[\n\t]/.test(text.slice(originalSplitFrom ?? originalStart, originalEnd))) return; parts.push(text.slice(copied, originalStart), replacement); copied = originalEnd; }); @@ -343,7 +372,9 @@ function redactSplitCredentials(text: string): string { }, /\w/, ); - const schemes = keys.replace(SCHEME_CREDENTIAL, '$1 [redacted]'); + // A scheme label may take a URL scheme as its token, and a URL without its scheme hides its + // userinfo from every later rule, so the userinfo is masked first, as in the main chain. + const schemes = redactUrlUserinfo(keys).replace(SCHEME_CREDENTIAL, '$1 [redacted]'); return redactJoinedCredentialSpans( schemes, (joined, mask) => { @@ -366,14 +397,9 @@ function redactSplitCredentials(text: string): string { * no-backtracking-RegExp / no-ReDoS posture. */ export function redactSecretShapedText(text: string): string { - // One pass can change the context a later pass decides on (a masked query value can remove the - // authority's last `@`), so the projection repeats until it is stable within three passes and - // fails closed otherwise: its output is stable under another projection. - const once = redactSecretShapedOnce(text); - if (once === text) return once; - const twice = redactSecretShapedOnce(once); - if (twice === once) return once; - return redactSecretShapedOnce(twice) === twice ? twice : '[redacted]'; + // One pass can change the context a later pass decides on, so the projection repeats until it is + // stable and fails closed otherwise: its output is stable under another projection. + return toFixedPoint(redactSecretShapedOnce, text, '[redacted]'); } function redactSecretShapedOnce(text: string): string { @@ -390,20 +416,8 @@ function redactSecretShapedOnce(text: string): string { // one simple greedy run. V8 can exhaust its RegExp stack on an admitted-scale `{N,}` run; the // equivalent `(?=class{N})class+` form keeps the whole value instead of clipping a credential. .replace(SCHEME_CREDENTIAL, '$1 [redacted]') - // A secret-ish key (bounded wrappers keep this ReDoS-safe) + `=`/`:` + its value. The optional `["']?` - // BEFORE the separator catches the JSON `"access_token":"…"` shape (an OAuth/egress response body). The - // value has QUOTED and UNQUOTED branches. A quoted value ends at the first UNESCAPED matching quote, - // including JSON's escaped quotes/backslashes; otherwise a valid password such as `a\"secret-tail` - // left its tail exposed. The two alternatives inside each repetition are disjoint: a backslash - // consumes exactly the following non-newline character, while the ordinary class excludes it. - // No ambiguous nested repetition/backref is needed. Unquoted values retain their delimiter and - // minimum-length policy. Kept a single analyzable LITERAL (not composed via `new RegExp`) so Sonar's static - // super-linear-runtime (S5852) check still covers this security-critical pattern; its keyword-alternation - // breadth is deliberate (and `apikey` is dropped — `api[_-]?key` already subsumes it). - .replace( - /\b[\w-]{0,32}(?:password|passwd|secret|token|api[_-]?key|authorization|access[_-]?key|private[_-]?key|client[_-]?secret)[\w-]{0,16}["']?\s*[=:]\s*(?:"(?:[^"\\\r\n]|\\[^\r\n])*"|'(?:[^'\\\r\n]|\\[^\r\n])*'|(?=[^\s"',;&]{6})[^\s"',;&]+)/gi, - '[redacted]', - ); + // A secret-ish key and its value (`SECRET_KEY_VALUE`). + .replace(SECRET_KEY_VALUE, '[redacted]'); // Well-known standalone credential shapes (OpenAI, Stripe, AWS incl. STS ASIA/ABIA, GitHub token + PAT, // GitLab PAT, Slack, Google API + OAuth, HuggingFace, npm, JWT). Kept as ONE alternation ON PURPOSE: a // single leftmost-longest scan is required so two ADJACENT / EMBEDDED credential shapes (e.g. a JWT whose @@ -458,9 +472,11 @@ const HOST_SHAPED_MAX_LENGTH = 260; * tail in the durable digest. Paths and JSON delimiters stop the scan; queries/fragments * after an existing userinfo separator remain public. Before that separator, a raw ?/# * begins public query/fragment text only when the text before it is host-shaped **and** an - * `=` assignment follows it; anything else may be a malformed username or password, and unless a - * host-shaped name follows the last `@` such an ambiguous authority is masked through its end. This - * is the same predicate and extent as the LLM diagnostic backstop (`packages/llm/src/scrub-secrets.ts`). + * `=` assignment follows it; anything else may be a malformed username or password. This is the + * same predicate as the LLM diagnostic backstop (`packages/llm/src/scrub-secrets.ts`). For such an + * ambiguous userinfo the backstop masks through the authority's end, while this projection, whose rules + * run in sequence, masks it with the query values its later rules would mask whole (see + * `laterQueryValueEnd`). * The projection differs from that backstop by design: its authority also ends at `"`, `'`, `,`, * `}` and `]`, so userinfo containing one of them (an already-masked `[REDACTED]` password among * them) is left as it is; it recognizes any scheme of up to 32 characters rather than a finite list; @@ -474,20 +490,39 @@ function isPublicQueryContact(userinfo: string, query: number): boolean { return HOST_SHAPED_AUTHORITY.test(userinfo.slice(0, query)) && userinfo.includes('=', query + 1); } -/** Whether an authority's text after its last `@` begins with a host, up to any query or fragment. */ -function hostFollows(rest: string): boolean { - const end = rest.search(/[?#]/); - const host = end < 0 ? rest : rest.slice(0, end); - // Bounded like the contact prefix, so an admitted-size run cannot exhaust the RegExp stack. - return host.length <= HOST_SHAPED_MAX_LENGTH && HOST_SHAPED_AUTHORITY.test(host); +/** A query assignment a later rule may read: a name and `=` after a raw `?`, `&` or `#`. */ +const QUERY_ASSIGNMENT = /[?&#]([\w.-]{1,64})=/g; + +/** + * Where core's later rules would end their masks of the query values that start inside an ambiguous + * userinfo (`from`..`to`), or -1: the opaque-query rule takes a value of 24 or more characters after + * `?`/`&`, and the secret key=value rule a secret-ish name's value. Masking the userinfo first would + * remove the names those rules read, so its mask runs to the same end and never farther, leaving + * every later credential's context in place. + */ +function laterQueryValueEnd(text: string, from: number, to: number): number { + let end = -1; + const opaqueEnd = /[\s&#"',}\]]/g; + for (const match of text.slice(from, to).matchAll(QUERY_ASSIGNMENT)) { + const valueStart = from + match.index + match[0].length; + if (match[0][0] !== '#') { + opaqueEnd.lastIndex = valueStart; + const valueEnd = opaqueEnd.exec(text)?.index ?? text.length; + if (valueEnd - valueStart >= 24) end = Math.max(end, valueEnd); + } + // The key=value rule may start at any word boundary inside the name, before its keyword. + const keyAt = (match[1] ?? '').search(SECRET_QUERY_NAME); + if (keyAt < 0) continue; + SECRET_KEY_VALUE_AT.lastIndex = from + match.index + 1 + keyAt; + if (SECRET_KEY_VALUE_AT.test(text)) end = Math.max(end, SECRET_KEY_VALUE_AT.lastIndex); + } + return end; } -function visitUrlUserinfo(text: string, mask: SecretSpanVisitor): void { +function visitUrlUserinfo(text: string, mask: SecretSpanVisitor, extend = false): void { if (!text.includes('://')) return; const scheme = /[a-z][\w+.-]{0,31}:\/\//gi; const delimiter = /[/\s"',}\]]/g; - // The LLM backstop's authority rest: `]` stays inside it, so a bracketed IP host stays whole. - const restDelimiter = /[/\s"',}]/g; for (let match = scheme.exec(text); match !== null; match = scheme.exec(text)) { const start = scheme.lastIndex; delimiter.lastIndex = start; @@ -506,15 +541,14 @@ function visitUrlUserinfo(text: string, mask: SecretSpanVisitor): void { const hasPassword = colon >= 0 && colon < userinfo.length - 1; if (!hasPassword && userinfo.length < 16) continue; let maskEnd = start + at; - if (query >= 0) { - // A raw ?/# leaves the authority ambiguous: unless a host follows its last `@`, the text there - // may be the rest of a query value (`https://svc?password=P@ss`), so the mask runs through it. - restDelimiter.lastIndex = maskEnd + 1; - const restEnd = restDelimiter.exec(text)?.index ?? text.length; - if (!hostFollows(text.slice(maskEnd + 1, restEnd))) maskEnd = restEnd; + // A raw ?/# leaves the authority ambiguous: the userinfo may also be the query a later rule reads + // (`https://svc?password=P@ss`). Its mask also covers each value that rule would mask whole. The + // joined view (no `extend`) masks the userinfo only, so no value runs across a removed LF/TAB. + if (extend && query >= 0) { + maskEnd = Math.max(maskEnd, laterQueryValueEnd(text, start + query, start + at)); } - mask(start, maskEnd, '[redacted]'); - // Skip the recognized userinfo without hiding a later scheme in the public host/tail. + mask(start, maskEnd, '[redacted]', match.index); + // Skip the masked span without hiding a later scheme in the public host/tail. scheme.lastIndex = Math.max(maskEnd, start + at + 1); } } @@ -522,10 +556,14 @@ function visitUrlUserinfo(text: string, mask: SecretSpanVisitor): void { function redactUrlUserinfo(text: string): string { const parts: string[] = []; let cursor = 0; - visitUrlUserinfo(text, (start, end, replacement) => { - parts.push(text.slice(cursor, start), replacement); - cursor = end; - }); + visitUrlUserinfo( + text, + (start, end, replacement) => { + parts.push(text.slice(cursor, start), replacement); + cursor = end; + }, + true, + ); if (parts.length === 0) return text; parts.push(text.slice(cursor)); return parts.join(''); diff --git a/packages/core/src/tools/fixed-point.test.ts b/packages/core/src/tools/fixed-point.test.ts new file mode 100644 index 00000000..9e601bf6 --- /dev/null +++ b/packages/core/src/tools/fixed-point.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest'; + +import { toFixedPoint } from './fixed-point.js'; + +describe('toFixedPoint', () => { + const failed = '[failed]'; + const passes = (outputs: readonly string[]) => { + const calls: string[] = []; + const pass = (text: string) => { + calls.push(text); + return outputs[calls.length - 1] ?? text; + }; + return { pass, calls }; + }; + + it('returns an input the first pass leaves unchanged after one pass', () => { + const { pass, calls } = passes(['a']); + expect(toFixedPoint(pass, 'a', failed)).toBe('a'); + expect(calls).toEqual(['a']); + }); + + it('returns the first output once a second pass confirms it', () => { + const { pass, calls } = passes(['b', 'b']); + expect(toFixedPoint(pass, 'a', failed)).toBe('b'); + expect(calls).toEqual(['a', 'b']); + }); + + it('returns the second output once a third pass confirms it', () => { + const { pass, calls } = passes(['b', 'c', 'c']); + expect(toFixedPoint(pass, 'a', failed)).toBe('c'); + expect(calls).toEqual(['a', 'b', 'c']); + }); + + it('fails closed when the third pass still changes the text', () => { + const { pass, calls } = passes(['b', 'c', 'd']); + expect(toFixedPoint(pass, 'a', failed)).toBe(failed); + expect(calls).toEqual(['a', 'b', 'c']); + }); +}); diff --git a/packages/core/src/tools/fixed-point.ts b/packages/core/src/tools/fixed-point.ts new file mode 100644 index 00000000..57cd3b5d --- /dev/null +++ b/packages/core/src/tools/fixed-point.ts @@ -0,0 +1,13 @@ +/** + * Repeats one projection pass until its output stops changing, at most three passes. A pass can + * change the context the next pass decides on, so a settled output is a fixed point, and an output + * that still changes on the third pass fails closed to `failed`. Callers choose a `failed` value the + * pass leaves unchanged, so the result is stable under another call either way. + */ +export function toFixedPoint(pass: (text: string) => string, text: string, failed: string): string { + const once = pass(text); + if (once === text) return once; + const twice = pass(once); + if (twice === once) return once; + return pass(twice) === twice ? twice : failed; +} diff --git a/packages/llm/src/fixed-point.test.ts b/packages/llm/src/fixed-point.test.ts new file mode 100644 index 00000000..b8dd9759 --- /dev/null +++ b/packages/llm/src/fixed-point.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest'; + +import { toFixedPoint } from './fixed-point.js'; + +describe('toFixedPoint', () => { + const failed = '[FAILED]'; + const passes = (outputs: readonly string[]) => { + const calls: string[] = []; + const pass = (text: string) => { + calls.push(text); + return outputs[calls.length - 1] ?? text; + }; + return { pass, calls }; + }; + + it('returns an input the first pass leaves unchanged after one pass', () => { + const { pass, calls } = passes(['a']); + expect(toFixedPoint(pass, 'a', failed)).toBe('a'); + expect(calls).toEqual(['a']); + }); + + it('returns the first output once a second pass confirms it', () => { + const { pass, calls } = passes(['b', 'b']); + expect(toFixedPoint(pass, 'a', failed)).toBe('b'); + expect(calls).toEqual(['a', 'b']); + }); + + it('returns the second output once a third pass confirms it', () => { + const { pass, calls } = passes(['b', 'c', 'c']); + expect(toFixedPoint(pass, 'a', failed)).toBe('c'); + expect(calls).toEqual(['a', 'b', 'c']); + }); + + it('fails closed when the third pass still changes the text', () => { + const { pass, calls } = passes(['b', 'c', 'd']); + expect(toFixedPoint(pass, 'a', failed)).toBe(failed); + expect(calls).toEqual(['a', 'b', 'c']); + }); +}); diff --git a/packages/llm/src/fixed-point.ts b/packages/llm/src/fixed-point.ts new file mode 100644 index 00000000..e8dde636 --- /dev/null +++ b/packages/llm/src/fixed-point.ts @@ -0,0 +1,14 @@ +/** + * Repeats one masking pass until its output stops changing, at most three passes. A pass can glue a + * marker to the text after another rule's token, which the next pass reads anew, so a settled output + * is a fixed point, and an output that still changes on the third pass fails closed to `failed`. + * Callers choose a `failed` value the pass leaves unchanged, so the result is stable under another + * call either way. Dependency-free, so the pure `scrub-secrets` leaf can load it. + */ +export function toFixedPoint(pass: (text: string) => string, text: string, failed: string): string { + const once = pass(text); + if (once === text) return once; + const twice = pass(once); + if (twice === once) return once; + return pass(twice) === twice ? twice : failed; +} diff --git a/packages/llm/src/scrub-secrets.test.ts b/packages/llm/src/scrub-secrets.test.ts index ef1abde4..ee7a6b5c 100644 --- a/packages/llm/src/scrub-secrets.test.ts +++ b/packages/llm/src/scrub-secrets.test.ts @@ -278,6 +278,9 @@ describe('PR95 B2 split provider/header and malformed userinfo diagnostics', () for (let index = 1; index < name.length; index += 1) { const input = `header ${name.slice(0, index)}${separator}${name.slice(index)}: opaque-Z29; retry later`; expectCredentialDiagnostic(scrubSecrets(input)).toBe('[REDACTED]'); + // JSON-encoded JSON; the inner mask keeps the whitespace-stripped residual from answering. + const escaped = `{\\"${name.slice(0, index)}${separator}${name.slice(index)}\\":\\"[REDACTED]opaque-Z29\\"}`; + expectCredentialDiagnostic(scrubSecrets(escaped)).toBe('[REDACTED]'); } } } @@ -731,6 +734,15 @@ describe('PR95 review round 1: label stacking, host bounds and masked userinfo', expect(scrubSecrets('postgres://app:pa[REDACTED]ss@db.example')).toBe( 'postgres://[REDACTED]@db.example', ); + // Only an exactly masked password is kept: text glued after the marker, or an empty password, is not. + for (const [input, expected] of [ + ['postgres://app:[REDACTED]tail@db.example/x', 'postgres://[REDACTED]@db.example/x'], + ['redis://:[REDACTED]SECRETTAIL@cache:6379', 'redis://[REDACTED]@cache:6379'], + ['https://:[REDACTED]tail@h.example/', 'https://[REDACTED]@h.example/'], + ['https://[REDACTED]:@h.example/', 'https://[REDACTED]@h.example/'], + ] as const) { + expect(scrubSecrets(input), 'SECRETS_B5_MASKED_USERINFO').toBe(expected); + } }); }); @@ -1050,6 +1062,18 @@ describe('PR95 review round 4: concatenated labels, ambiguous authorities and a ]) { bound(scrubSecrets(input)).toBe('[REDACTED]'); } + // Any case, and labels split by LF or TAB, count alike. + for (const label of ['bearer', 'BASIC', 'B\nearer', 'B\tasic']) { + for (const count of [8, 9]) { + const run = label.repeat(count); + bound(scrubSecrets(`Authorization: Bearer ${run} ${token}`)).toBe( + count <= 8 ? `Authorization: Bearer ${run} [REDACTED]` : '[REDACTED]', + ); + bound(scrubSecrets(`api-key: ${run} ${token}`)).toBe( + count <= 8 ? `api-key: ${run} [REDACTED]` : '[REDACTED]', + ); + } + } }); it( @@ -1186,7 +1210,7 @@ describe('PR95 review round 4: concatenated labels, ambiguous authorities and a ); }); - it('reaches a fixed point, failing closed when it cannot', () => { + it('reaches a fixed point', () => { for (const [input, expected] of [ ['api-key: https://"@host.example', 'api-key: [REDACTED]'], ['{"x-api-key":"https://\\"@h.example"}', '{"x-api-key":"[REDACTED]"}'], @@ -1226,3 +1250,57 @@ describe('PR95 review round 4: concatenated labels, ambiguous authorities and a ADMITTED_SIZE_CASE_TIMEOUT_MS, ); }); + +describe('PR95 review round 5: label pieces, a split ftp credential and the repeated residual', () => { + const token = ['abcdef', 'ghijkl', '123456'].join(''); + const cases = (marker: string, pairs: readonly (readonly [string, string])[]) => { + for (const [input, expected] of pairs) { + const output = scrubSecrets(input); + expect(output, marker).toBe(expected); + expect(output.includes(token) || output.includes('SECRET'), marker).toBe(false); + expect(scrubSecrets(output), marker).toBe(output); + } + }; + + it('ends a standalone token before a trailing delimiter, a quote-closed label or a concatenated run', () => { + cases('SECRETS_B5_LABEL_PIECES', [ + [`Authorization: Bearer ${token}; retry`, 'Authorization: Bearer [REDACTED]; retry'], + [`Authorization: Bearer ${token}, retry`, 'Authorization: Bearer [REDACTED], retry'], + [`Authorization: Basic ${token}; retry`, 'Authorization: Basic [REDACTED]; retry'], + [`Authorization: Basic ${token}, retry`, 'Authorization: Basic [REDACTED], retry'], + [ + `Authorization: Bearer x("Bearer" ${token}`, + 'Authorization: Bearer [REDACTED]"Bearer" [REDACTED]', + ], + [ + `Authorization: Bearer x(${'Bearer'.repeat(8)} ${token}`, + `Authorization: Bearer [REDACTED]${'Bearer'.repeat(8)} [REDACTED]`, + ], + ]); + }); + + it('blanks an LF/TAB-split ftp credential after a line that ends in s', () => { + cases('SECRETS_B5_SPLIT_SFTP', [ + ['Uploading files\nftp://deploySECRET\n1234@host.example/x', '[REDACTED]'], + ['Uploading files\n\tftp://deploySECRET\n1234@host.example/x', '[REDACTED]'], + ['files\nf\ntp://TOKENSECRET@host', '[REDACTED]'], + ]); + // A real sftp scheme keeps its colonless user, also when the user is split. + for (const input of [ + 'files\nsftp://deploy\nuser@host', + 'connect to sftp://deploy@files.example', + ]) { + expect(scrubSecrets(input), 'SECRETS_B5_SPLIT_SFTP').toBe(input); + } + }); + + it('blanks a diagnostic whose key prefix reaches its length across spaces after a mask', () => { + // The repeat reads another rule's mask as a segment start for the whitespace-stripped residual. + for (const input of [ + 'Authorization: Bearer xyz sk-ant api key invalid for this org', + 'x-api-key: abc AIza prefix keys are restricted here', + ]) { + expect(scrubSecrets(input), 'SECRETS_B5_RESIDUAL_AFTER_MASK').toBe('[REDACTED]'); + } + }); +}); diff --git a/packages/llm/src/scrub-secrets.ts b/packages/llm/src/scrub-secrets.ts index 3cbedf83..1a45932f 100644 --- a/packages/llm/src/scrub-secrets.ts +++ b/packages/llm/src/scrub-secrets.ts @@ -1,5 +1,6 @@ import { stripTerminalControls } from '@relavium/shared/terminal-safe'; +import { toFixedPoint } from './fixed-point.js'; import { splitProbe, type SplitProbe, type SplitProbeAnchor } from './split-credential-probe.js'; /** The marker every masked span becomes; a diagnostic that cannot be masked in place becomes it whole. */ @@ -26,11 +27,7 @@ const MASKED = '[REDACTED]'; * The canonical policy is error-handling.md's diagnostic secret backstop. */ export function scrubSecrets(text: string): string { - const once = scrubOnce(text); - if (once === text) return once; - const twice = scrubOnce(once); - if (twice === once) return once; - return scrubOnce(twice) === twice ? twice : MASKED; + return toFixedPoint(scrubOnce, text, MASKED); } /** A range of the original diagnostic that one rule masks with `[REDACTED]`. */ @@ -315,7 +312,8 @@ function applyRule(text: string, rule: SecretRule): string { * that prefilter skips only impossible starts. * Keys have one token run after a bounded prefix/minimum. Header labels are bounded and their * trailing whitespace cannot contain another label. URL authority excludes slash, so a later URL - * prefix delimits a failed authority scan. Successful matches advance both cursors over their span; + * prefix delimits a failed authority scan. Successful matches advance both cursors over their span, + * except a kept candidate whose first letter ends a line, which advances one start, once per start; * no failed candidate rescans an admitted-size run at every LF/TAB boundary. * Opaque Bearer/Basic values and literal query names keep their established delimiter policy. */ @@ -366,6 +364,18 @@ function hasSplitCredential(text: string, joined: string, rule: SecretRule): boo pattern.lastIndex = joinedOffset; const match = pattern.exec(joined); if (match !== null) { + // A candidate the policy keeps (a connection string's user, a public contact) whose first + // letter ends a line may be a word glued across the break to a shorter listed scheme + // (`files\nftp://` reads as `sftp://`), so the next start is tried instead of its span. + if ( + /[\n\t]/.test(text[originalOffset + 1] ?? '') && + rule.credential !== undefined && + !rule.credential(match[0]) + ) { + originalOffset += 1; + joinedOffset += 1; + continue; + } const start = originalOffset; const joinedStart = joinedOffset; while (joinedOffset < pattern.lastIndex) { diff --git a/tools/causal-controls/provenance.test.mjs b/tools/causal-controls/provenance.test.mjs index e49bcb1a..59ed7d0d 100644 --- a/tools/causal-controls/provenance.test.mjs +++ b/tools/causal-controls/provenance.test.mjs @@ -267,10 +267,10 @@ test('every declared filesystem and secrets anchor matches current production so }); test('secrets workload corrections retain independent complete-count and reader-witness pins', () => { - assert.equal(secretsControls.length, 109); + assert.equal(secretsControls.length, 110); assert.equal( secretsControls.reduce((count, control) => count + 1 + (control.witnesses ?? []).length, 0), - 481, + 483, ); for (const [id, file, expectedTests] of [ ['run-default-secret-reader', 'apps/cli/src/commands/run.test.ts', 84], @@ -313,12 +313,13 @@ test('secrets workload corrections retain independent complete-count and reader- 'core-userinfo-contact-host', 'core-userinfo-host-length', 'core-userinfo-host-anchors', - 'core-userinfo-authority-rest', - 'core-userinfo-rest-length', - 'core-scheme-url-lookahead', + 'core-userinfo-query-extent', + 'core-userinfo-joined-extent', + 'core-split-userinfo-before-scheme', + 'core-userinfo-split-scheme', 'core-userinfo-split-edges', 'core-fixed-point', - ].map((id) => [id, ['packages/core/src/tools/bounding.test.ts'], 162]), + ].map((id) => [id, ['packages/core/src/tools/bounding.test.ts'], 167]), ...[ 'diagnostic-joined-before-masking', 'diagnostic-joined-original-boundary', @@ -356,7 +357,7 @@ test('secrets workload corrections retain independent complete-count and reader- 'diagnostic-mask-merge', 'diagnostic-token-only-mask', // Round 4: one label rule for both labels, and each new leak or totality mechanism. - ...['concat-run', 'concat-stop', 'concat-refusal', 'start-bound', 'lead', 'run-bound'].map( + ...['concat-run', 'concat-stop', 'start-bound', 'lead', 'run-bound'].map( (mechanism) => `diagnostic-label-${mechanism}`, ), 'diagnostic-header-concat-run', @@ -367,7 +368,8 @@ test('secrets workload corrections retain independent complete-count and reader- 'diagnostic-fixed-point', 'diagnostic-userinfo-authority-rest', 'diagnostic-userinfo-rest-length', - ].map((id) => [id, ['packages/llm/src/scrub-secrets.test.ts'], 212]), + 'diagnostic-url-split-retry', + ].map((id) => [id, ['packages/llm/src/scrub-secrets.test.ts'], 215]), ...['diagnostic-inline-projection-order', 'diagnostic-original-control-boundary'].map((id) => [ id, ['apps/cli/src/render/sanitize.test.ts', 'apps/cli/src/process/render-error.test.ts'], diff --git a/tools/secrets-controls/README.md b/tools/secrets-controls/README.md index 09ec6ece..28c61d88 100644 --- a/tools/secrets-controls/README.md +++ b/tools/secrets-controls/README.md @@ -4,7 +4,7 @@ Step 4 first-run input matrix, Step 5 snapshot/export, Step 7 provider-header and PR #95 B2 redaction controls on physical source copies. `--mechanism ` supports diagnosis only; selected runs retain `complete: false`. The current manifest -declares **109 independent controls and 481 required named assertion witnesses**. +declares **110 independent controls and 483 required named assertion witnesses**. These counts describe the workload; a declaration is not evidence that its removal passed. Actual fixed/removal/restored results belong to the coordinator's summary. @@ -178,8 +178,8 @@ modern `sk-` body characters, joined-span application, the original-text left-bo check, split scheme labels, split API-header labels and raw `?`/`#` password recognition independently. Four LLM controls remove the before-masking LF/TAB probe, its original left-boundary check, split API-header recognition and raw-password recognition. -Every core phase executes the complete 162-case bounding file; every new LLM phase -executes the complete 212-case scrubber file. Independent ordinary tooling tests pin +Every core phase executes the complete 167-case bounding file; every new LLM phase +executes the complete 215-case scrubber file. Independent ordinary tooling tests pin these inventories, the 278-case overlap cohort, 180-case CLI diagnostic cohort and 18-case command-export cohort. Actual collection still has to match in every phase. @@ -260,14 +260,14 @@ previous-word prefilter and the rule's own boundary predicate; retaining either would leave the tested policy active. Header/key recovery removals still change their original-rule boundary only, and all removals retain the same canonical matchers and bounded raw/joined standalone visitors. Current collection is 25 -complete files / 1,465 cases, with each control's exact workload and named target +complete files / 1,473 cases, with each control's exact workload and named target checked separately. These are updated declarations and author checks, not new physical causal captures or a clean review round. R3 is preserved as NONCLEAN; later corrective acceptance belongs to the owning phase register. Prior selected `7e9096a8` captures retain their older source/workload attribution. -The current 109-control / 327-phase full declaration projects 1,969 retained files -(109 × 18 + 7) under the unchanged 2,000-file / 64-MiB export bounds. This is a capacity +The current 110-control / 330-phase full declaration projects 1,987 retained files +(110 × 18 + 7) under the unchanged 2,000-file / 64-MiB export bounds. This is a capacity calculation, not a measured full capture. Historical 41-control captures retain their original source and cohort attribution; selected B2 or original-boundary recovery captures are not full-family acceptance. @@ -326,11 +326,34 @@ residual, where a label rule (which needs whitespace after the label) never matc bound stays controlled through the residual of a separate-label run. Edges that only change how much is masked, never whether a token leaks (a word-glued label is no label, whitespace before a header colon, a host after an ambiguous authority staying visible), are pinned by tests without a -control, which keeps the family inside the 2,000-file export bound. Two quadratic guards (the label -token's first-character lookahead and the header stop's bounded escape run) and the URL rule's -`://` prefilter are output-equivalent and pinned only by the admitted-size case deadline: the -coordinator counts a timeout as infrastructure, never as a removal's failure, so they have no -control. +control, which keeps the family inside the 2,000-file export bound. Three guards are pinned only +by the admitted-size case deadline: the label token's first-character lookahead and the URL rule's +`://` prefilter are output-equivalent, and the header stop's bounded escape run only decides how +many escapes past fifteen fold into the mask; removing any of them brings back a quadratic scan, +and the coordinator counts a timeout as infrastructure, never as a removal's failure, so they have +no control. + +The fifth round found that core's round-4 authority extent erased the context its later rules +read: core's rules run in sequence, so masking through an ambiguous authority's end swallowed the +key name, label, scheme or PEM marker of the next credential, and in the LF/TAB-joined view it ran +across a real line break. Core now masks an ambiguous userinfo together with exactly the query +values its opaque-query and secret key/value rules would mask whole, ending where those rules end +them, and masks the userinfo only in the joined view; the URL-scheme lookahead after a scheme label +is gone, because the split view now masks URL userinfo before scheme labels, as the main pass +always did. Three core controls were replaced by three for those mechanisms +(`core-userinfo-query-extent`, `core-userinfo-joined-extent`, +`core-split-userinfo-before-scheme`); `core-userinfo-split-scheme` controls a separator inside a +URL's `scheme://`, which now splits its userinfo as one at the userinfo's edges does (core had left +`https:/\n/user:pass@host` unmasked since before the increment); and `diagnostic-url-split-retry` +controls the split scan's retry of a kept candidate glued across a line break (`files\nftp://` +read as `sftp://`). `diagnostic-label-concat-refusal` is retired to make room, and its test stays. +The family is now 110 controls, the most the unchanged 2,000-file export bound admits (110 × 18 + 7 += 1,987). From this round on, a control pins each new leak-bearing mechanism, and the pieces inside +a mechanism's pattern (one character of a delimiter class, a repetition bound, a flag, a fixed +point's fail-closed branch) are pinned by discriminating tests instead: the fifth round's tests for +the label rule's trailing `;`/`,`, its embedded stop's closing quote run and bound, the nine-label +bound's case and LF/TAB letters, the exactly masked password, the escaped joined header name and +both fixed-point helpers were each checked against the mutation they kill. The thin entry point uses the same coordinator, outcome and bounded process join as the filesystem family in [causal-controls](../causal-controls/README.md). diff --git a/tools/secrets-controls/mutations.mjs b/tools/secrets-controls/mutations.mjs index ba72c9cc..ffb0bcb4 100644 --- a/tools/secrets-controls/mutations.mjs +++ b/tools/secrets-controls/mutations.mjs @@ -148,21 +148,8 @@ const overlapMerge = ' if (open !== undefined && range.start < open.end) {'; const mergeExtent = ' open.end = Math.max(open.end, range.end);'; const hostFollowsLine = ' return host.length <= HOST_SHAPED_MAX_LENGTH && HOST_SHAPED_AUTHORITY.test(host);'; -const llmFixedPoint = [ - ' const once = scrubOnce(text);', - ' if (once === text) return once;', - ' const twice = scrubOnce(once);', - ' if (twice === once) return once;', - ' return scrubOnce(twice) === twice ? twice : MASKED;', -].join('\n'); -const coreFixedPoint = [ - ' const once = redactSecretShapedOnce(text);', - ' if (once === text) return once;', - ' const twice = redactSecretShapedOnce(once);', - ' if (twice === once) return once;', - " return redactSecretShapedOnce(twice) === twice ? twice : '[redacted]';", -].join('\n'); -const coreSchemeCredential = String.raw` /\b(b[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|b[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c|t[\n\t]*o[\n\t]*k[\n\t]*e[\n\t]*n)\s+(?=[\w.~+/-]{8})(?![\w.~+/-]*:\/\/)[\w.~+/-]+={0,2}/gi;`; +const llmFixedPoint = ' return toFixedPoint(scrubOnce, text, MASKED);'; +const coreFixedPoint = " return toFixedPoint(redactSecretShapedOnce, text, '[redacted]');"; const headerTokenMask = ' mask: tokenGroup,\n joinedPattern:'; const stackedLabelCase = 'masks a header token after up to eight stacked labels and fails closed beyond them'; @@ -343,7 +330,8 @@ function redactJoinedCredentialSpans( } return target + removed; } - visit(joined, (start, end, replacement) => { + visit(joined, (start, end, replacement, splitFrom) => { + const originalSplitFrom = splitFrom === undefined ? undefined : advance(splitFrom); const originalStart = advance(start, !ownsEdges); const originalEnd = advance(end, ownsEdges); if (boundary?.test(text[originalStart - 1] ?? '')) return;`; @@ -354,7 +342,7 @@ const withoutCoreOriginalBoundary = coreOriginalBoundary ) .replace(" if (boundary?.test(text[originalStart - 1] ?? '')) return;", ''); /** The complete core projection test file every core projection control runs. */ -const CORE_PROJECTION_CASES = 162; +const CORE_PROJECTION_CASES = 167; function coreProjectionControl(id, before, after, targets, marker = coreProjectionMarker) { return control( id, @@ -369,7 +357,7 @@ function coreProjectionControl(id, before, after, targets, marker = coreProjecti ); } /** The complete scrubber test file every LLM diagnostic control runs. */ -const LLM_SCRUB_CASES = 212; +const LLM_SCRUB_CASES = 215; function diagnosticControl(id, before, after, targets, marker = diagnosticMarker) { return control( id, @@ -1061,8 +1049,8 @@ export const mutations = [ ), coreProjectionControl( 'core-joined-credential-spans', - ' visit(joined, (start, end, replacement) => {', - " visit('', (start, end, replacement) => {", + ' visit(joined, (start, end, replacement, splitFrom) => {', + " visit('', (start, end, replacement, splitFrom) => {", [ 'masks every LF/TAB split without losing the surrounding public identity', ...splitCoreHeaders, @@ -1076,8 +1064,8 @@ export const mutations = [ ), coreProjectionControl( 'core-split-scheme-label', - coreSchemeCredential, - String.raw` /\b(bearer|basic|token)\s+(?=[\w.~+/-]{8})(?![\w.~+/-]*:\/\/)[\w.~+/-]+={0,2}/gi;`, + String.raw`/\b(b[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|b[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c|t[\n\t]*o[\n\t]*k[\n\t]*e[\n\t]*n)\s+(?=[\w.~+/-]{8})[\w.~+/-]+={0,2}/gi`, + String.raw`/\b(bearer|basic|token)\s+(?=[\w.~+/-]{8})[\w.~+/-]+={0,2}/gi`, splitCoreHeaders, ), coreProjectionControl( @@ -1175,7 +1163,6 @@ export const mutations = [ ...[ ['diagnostic-label-concat-run', { concat: '' }], ['diagnostic-label-concat-stop', { stop: label.stop.replace(label.concat, '') }], - ['diagnostic-label-concat-refusal', { notLabel: label.notLabel.replace('{1,8}', '{1}') }], ].map(([id, part]) => diagnosticControl( id, @@ -1362,7 +1349,7 @@ export const mutations = [ 'diagnostic-fixed-point', llmFixedPoint, ' return scrubOnce(text);', - ['reaches a fixed point, failing closed when it cannot'], + ['reaches a fixed point'], 'SECRETS_B4_FIXED_POINT', ), diagnosticControl( @@ -1465,26 +1452,37 @@ export const mutations = [ ['bounds the host test after an ambiguous authority at admitted size'], 'SECRETS_B4_REST_LENGTH', ), + // Round 5: an ambiguous userinfo is masked with exactly the query values core's later rules read. coreProjectionControl( - 'core-userinfo-authority-rest', - ' if (!hostFollows(text.slice(maskEnd + 1, restEnd))) maskEnd = restEnd;', + 'core-userinfo-query-extent', + ' maskEnd = Math.max(maskEnd, laterQueryValueEnd(text, start + query, start + at));', '', - ['masks a secret query value that holds a raw @ after a single-label host'], - 'SECRETS_B4_CORE_AUTHORITY_REST', + ['masks an ambiguous userinfo with the query values its later rules would mask whole'], + 'SECRETS_B5_CORE_QUERY_EXTENT', ), coreProjectionControl( - 'core-userinfo-rest-length', - hostFollowsLine, - ' return HOST_SHAPED_AUTHORITY.test(host);', - ['bounds the host test after an ambiguous authority at admitted size'], - 'SECRETS_B4_CORE_REST_LENGTH', + 'core-userinfo-joined-extent', + ' if (extend && query >= 0) {', + ' if (query >= 0) {', + ['keeps the context of a later credential after an ambiguous authority'], + 'SECRETS_B5_CORE_LATER_CONTEXT', + ), + coreProjectionControl( + 'core-split-userinfo-before-scheme', + " const schemes = redactUrlUserinfo(keys).replace(SCHEME_CREDENTIAL, '$1 [redacted]');", + " const schemes = keys.replace(SCHEME_CREDENTIAL, '$1 [redacted]');", + ['masks the userinfo of a URL behind a scheme label, split or not, and a token glued to a URL'], + 'SECRETS_B5_CORE_SCHEME_ORDER', ), coreProjectionControl( - 'core-scheme-url-lookahead', - coreSchemeCredential, - coreSchemeCredential.replace(String.raw`(?![\w.~+/-]*:\/\/)`, ''), - ['never takes a URL scheme as the token after a scheme label'], - 'SECRETS_B4_CORE_SCHEME_TOKEN', + 'core-userinfo-split-scheme', + " mask(start, maskEnd, '[redacted]', match.index);", + " mask(start, maskEnd, '[redacted]');", + ['\n', '\t'].map( + (separator) => + `masks URL userinfo whose scheme is split by an LF or TAB (${JSON.stringify(separator)})`, + ), + 'SECRETS_B5_CORE_SPLIT_SCHEME', ), coreProjectionControl( 'core-userinfo-split-edges', @@ -1500,9 +1498,16 @@ export const mutations = [ 'core-fixed-point', coreFixedPoint, ' return redactSecretShapedOnce(text);', - ['reaches a fixed point, failing closed when it cannot'], + ['reaches a fixed point'], 'SECRETS_B4_CORE_FIXED_POINT', ), + diagnosticControl( + 'diagnostic-url-split-retry', + " /[\\n\\t]/.test(text[originalOffset + 1] ?? '') &&", + ' false &&', + ['blanks an LF/TAB-split ftp credential after a line that ends in s'], + 'SECRETS_B5_SPLIT_SFTP', + ), // PR95: connection-string schemes join the finite URL list. diagnosticControl( 'diagnostic-connection-scheme-userinfo', From b89ac62e823a28e3ca0a031147179224fb22c5b9 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Tue, 20 Oct 2026 17:12:04 +0300 Subject: [PATCH 15/97] fix(core): address the sixth PR95 review round Round 6 of the PR95 increment's review (at 6b2e94dd) confirmed 21 findings, nine high. Round 5's value mirror for an ambiguous userinfo was quadratic, still masked through a later credential's key, label or PEM marker, and was dropped by the split view. All are fixed. Core projection (packages/core/src/tools/bounding.ts): - Core now works like the scrubber: every rule reports its ranges on one text and overlapping ranges merge before anything is replaced, so no mask removes the key name, scheme label, PEM marker or URL scheme another rule reads (R6-SCRUB-1, R6-SCRUB-3, SEC-R6-2, SEC-R6-3, GEN-R6-1). The value mirror (laterQueryValueEnd) and the split view's sequential rewrite are gone, which removes the quadratic scan (R6-SCRUB-2, CONS-R6-1, SEC-R6-1, GEN-R6-2). - A key or parameter in a URL userinfo's own text belongs to it: all of an unambiguous userinfo, and a user:password before a raw ?/#. The rest of an ambiguous userinfo is read by the key/value and opaque-query rules, so every value they mask whole is masked to its own end (R6-SCRUB-4), while a git remote such as https://x-access-token:@github.com/o/r keeps its host and path. - The key/value rule resumes its scan at each value, so a key inside a value is read too, and reads values through earlier masks, line breaks included; the scheme-label rule resumes after each label; a split header label, and a scheme label even where a key's value runs on over it, may start right after a credential. These keep the cases the old order masked by accident. - PEM blocks are found with one shared END search instead of a bounded rescan after every BEGIN marker (CRIT-R6-1): an open-PEM tool result at admitted size takes under a second, not 22 s. LLM backstop (packages/llm/src/scrub-secrets.ts): - The nine-label bound also reads the raw text, which normalization can shorten by one label letter after ESC [ (CRIT-R6-2). - The literal secret query names also follow #, as in an OAuth implicit-flow #access_token= (GEN-R6-3). Tests and controls: - Discriminating core cases for every round-6 witness, each prefix form of the round-5 witnesses, the userinfo's zones, nested keys and labels, values through masks, label boundaries, the PEM body bound and admitted-size linearity; LLM cases for the raw-text bound and fragments; and both maskers' fail-closed values, observed through a module mock of the fixed-point helper (TST-R6-1, TST-R6-2). Each was checked against the mutation it kills. - The secrets family stays at 110 controls (491 targets and witnesses): six core controls pin the new mechanisms, the three round-5 core controls go with their mechanisms, six are re-anchored, and three single-character pieces of the header rule are retired to make room after their tests were checked to kill the same mutations. Docs: built-in-tools.md (core's merged ranges and zones), effect-journal.md (the v5 projection changes, the split-scheme rule included), error-handling.md (core's sentence, fragments, the raw-text bound and two more limits), the controls README, the phase register's round table and witness count, and the status lines (DOC-R6-1..3, TST-R6-3, CONS-R6-2, GEN-R6-4). Refs: ADR-0113 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- CLAUDE.md | 2 +- docs/reference/shared-core/built-in-tools.md | 36 +- docs/reference/shared-core/effect-journal.md | 19 +- docs/roadmap/current.md | 2 +- .../phase-2.5.5-hardening-and-remediation.md | 8 +- docs/standards/error-handling.md | 15 +- packages/core/src/tools/bounding.test.ts | 270 +++++++++++ packages/core/src/tools/bounding.ts | 456 ++++++++++++------ packages/llm/src/scrub-secrets.test.ts | 43 +- packages/llm/src/scrub-secrets.ts | 11 +- tools/causal-controls/provenance.test.mjs | 19 +- tools/secrets-controls/README.md | 36 +- tools/secrets-controls/mutations.mjs | 142 +++--- 14 files changed, 791 insertions(+), 270 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 42b55d97..8c7daf07 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -126,7 +126,7 @@ gate (red since at least PR #90) was cleared on 2026-10-19. A pre-implementation and the maintainer's [2026-10-19 rulings](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) give every open PR #95 finding an owner and record 29 increments (no new ADR). The PR95 increment (the scrubber residuals, including both security residuals) is implemented on `development` and under -review: rounds 1–5 were NONCLEAN and are corrected ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)), and acceptance +review: rounds 1–6 were NONCLEAN and are corrected ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)), and acceptance needs two consecutive clean rounds. Then Step 9a.1 (release map/custody substrate, #46/ADR-0111). `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. diff --git a/CLAUDE.md b/CLAUDE.md index b4acc257..73de07db 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -215,7 +215,7 @@ gate (red since at least PR #90) was cleared on 2026-10-19. A pre-implementation and the maintainer's [2026-10-19 rulings](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) give every open PR #95 finding an owner and record 29 increments (no new ADR). The PR95 increment (the scrubber residuals, including both security residuals) is implemented on `development` and under -review: rounds 1–5 were NONCLEAN and are corrected ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)), and acceptance +review: rounds 1–6 were NONCLEAN and are corrected ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)), and acceptance needs two consecutive clean rounds. Then Step 9a.1 (release map/custody substrate, #46/ADR-0111). `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. diff --git a/docs/reference/shared-core/built-in-tools.md b/docs/reference/shared-core/built-in-tools.md index a1ab0576..4af9d1eb 100644 --- a/docs/reference/shared-core/built-in-tools.md +++ b/docs/reference/shared-core/built-in-tools.md @@ -99,13 +99,17 @@ with an optional port) **and** an `=` assignment follows it; otherwise the useri This is the same predicate as the [LLM diagnostic backstop](../../standards/error-handling.md#diagnostic-secret-backstop), including its 260-character bound on the host-shaped prefix. Masked userinfo that holds such a raw `?` or `#` -leaves the authority ambiguous: the userinfo may also be the query a later rule reads -(`https://svc?password=P@ss`). The backstop merges every rule's ranges and masks through the -authority's end; this projection's rules run in sequence, and masking the userinfo first would -remove the names its later rules read, so the userinfo mask also covers each query value that its -opaque-query rule (24 or more characters after `?`/`&`) or secret key/value rule would mask whole, -and ends where that rule would end it. It never masks farther, so the context of a credential after -the authority stays in place; in the LF/TAB-joined view it masks the userinfo only. The projection +leaves the authority ambiguous: the userinfo may also be a query that another rule reads +(`https://gateway?password=P@ss.word1`). The backstop masks through the authority's end unless a +host-shaped name follows its last `@`. This projection, like the backstop, finds every rule's +ranges on the same text and merges overlapping ranges before it replaces anything, so no rule's +mask removes the key name, scheme label, PEM marker or URL scheme another rule reads. Its +key/value and opaque-query rules read an ambiguous userinfo's keys and parameters from the raw `?` +or `#` on, or all of them when no `user:password` comes before it, and mask each value to its own +end (`https://gateway?password=P@ss.word1` → `https://[redacted]`). A key or parameter in a +userinfo's own text — all of an unambiguous userinfo, and a `user:password` before a raw `?` or +`#` — names a username or password, so its value does not run on through the public host and path +(`https://x-access-token:@github.com/o/r` → `https://[redacted]@github.com/o/r`). The projection otherwise differs from that backstop by design: its authority also ends at `"`, `'`, `,`, `}` and `]`, so userinfo that contains one of them — an already-masked `[REDACTED]` password among them — is left as it is; it recognizes any scheme of up to 32 characters rather than a finite list; and a @@ -114,12 +118,18 @@ under every scheme. LF/TAB inside a recognized token-shaped credential extends the masked original span. An ambiguous token-alphabet continuation after LF/TAB is conservatively masked with -that credential. An LF or TAB right after a URL's `://` or right before its userinfo's `@` -belongs to the split userinfo, which is masked as one span. URL userinfo is masked before an -authentication scheme label takes its token, in the split view as in the main pass, so a label -that takes a URL scheme (`token postgres://app:pw@db` → `token [redacted]://[redacted]@db`) never -hides a password. The projection repeats until its output stops changing, at most three passes, -and becomes `[redacted]` if it has not settled, so projecting a projection changes nothing. +that credential. An LF or TAB right after a URL's `://`, right before its userinfo's `@` or inside +its `scheme://` belongs to the split userinfo, which is masked as one span. A key's value runs +through every span an earlier rule masks, line breaks included, as through an applied mask, so a +quoted value that carries a split credential or a PEM block is masked whole. A key inside another +key's value, and a scheme label inside another label's token, start a credential of their own +(`token=a>password: ` → `[redacted]`; `Bearer a/Bearer ` → `Bearer [redacted] [redacted]`), +and an LF/TAB-split header label or a scheme label may start right after a credential or a PEM +block, as after an applied mask. A scheme label +that takes a URL scheme as its token no longer hides that URL's password +(`token postgres://app:pw@db` → `token [redacted]://[redacted]@db`). The projection repeats until +its output stops changing, at most three passes, and becomes `[redacted]` if it has not settled, so +projecting a projection changes nothing; every rule's scan stays linear in the admitted text. Ordinary spaces and punctuation separate public prose; the masker does not discard the whole diagnostic because a credential appears. Supported split shapes are recognized **before** complete-text one-line normalization, followed by diff --git a/docs/reference/shared-core/effect-journal.md b/docs/reference/shared-core/effect-journal.md index ee50c291..0126ad0c 100644 --- a/docs/reference/shared-core/effect-journal.md +++ b/docs/reference/shared-core/effect-journal.md @@ -370,8 +370,8 @@ inline-media redaction remain deliberate projection equality; raw credential byt added to the digest to distinguish calls. Recognized URL userinfo is masked through the authority's last `@`, preserving the host, -path, query and fragment; userinfo holding a raw `?` or `#` is masked together with the query -values the later rules would mask whole. Recognized opaque URL-query values are masked through their delimiter, including values +path, query and fragment; when it holds a raw `?` or `#`, the query values in it are masked to +their own ends as well. Recognized opaque URL-query values are masked through their delimiter, including values longer than 512 characters. Quoted secret-key values honor escaped quotes and backslashes through the actual closing quote. A recognized value must not leave a credential suffix in the projection. This remains a finite shape detector, not a claim to identify arbitrary @@ -379,12 +379,15 @@ unlabelled secrets. Existing old digests are not retroactively erased or rewritt A raw `?` or `#` before the authority's `@` is public only after a host-shaped prefix that an `=` assignment follows ([built-in tools](built-in-tools.md#finite-diagnostic-credential-masking)). -Wave 3's PR95 increment tightened that predicate without a new digest domain. It also masks -an ambiguous userinfo together with the query values the later rules would mask whole, masks -userinfo split by an LF or TAB at either edge, masks URL userinfo before an authentication -scheme label in the split view, and repeats the projection until it is stable. These change a -`v5:` projection only for the arguments they project differently, such as userinfo previously -treated as public: a resumed run whose +Wave 3's PR95 increment tightened that predicate without a new digest domain. It also merges +the ranges of every rule before replacing anything, so an ambiguous userinfo is masked together +with its query values and no mask removes another credential's context; reads a key or scheme +label inside another value as a credential of its own, and a value through earlier masks; masks +userinfo split by an LF or TAB at either edge or inside its `scheme://`; reads a split header +label or a scheme label right after a credential or a PEM block; and repeats the projection until +it is stable. +These change a `v5:` projection only for the arguments they project differently, such as +userinfo previously treated as public or a credential an earlier mask had hidden: a resumed run whose committed effect carried one now computes a different digest and refuses matching prepare instead of replaying, which is the fail-closed outcome of any digest difference. Every other `v5:` row keeps its digest. diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index f06a8b43..750d0813 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -165,7 +165,7 @@ Wave 3 implementation continues on `development`: Steps 1–8 are merged. A pre- of Steps 9–19 and the maintainer's rulings are recorded in the [2026-10-19 rulings](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) (no new ADR; 29 recorded increments). **The PR95 increment is implemented on `development` and under review** — the PR #95 -scrubber residuals, including the two security residuals; rounds 1–5 were NONCLEAN and are corrected, as its +scrubber residuals, including the two security residuals; rounds 1–6 were NONCLEAN and are corrected, as its [implementation checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20) records, and acceptance needs two consecutive clean rounds after the last material fix. **Step 9a.1** follows, the first increment of the release map/custody substrate (#46, ADR-0111), whose 9a.1–9d increments carry a dedicated archive/crypto security sitting. Steps 10–19 follow the diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 4ad5fd68..5c58675f 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -9423,8 +9423,9 @@ carries the same owners. `fixed-point.ts`) and in core's twin projection, `packages/core/src/tools/bounding.ts`, together with the CLI's authored-content detector, the secrets causal controls and the canonical documents. Base `9902e0a0`; commits `38e10d42` (Docs-0, the rulings), `dda3b45e` (implementation) and the -corrections `ef3ded0d`, `bdd292a6`, `6fb7e9d9`, `b2c50e88` and the round-5 commit that adds this -section. The commits stay local on `development` until the increment is accepted. +corrections `ef3ded0d`, `bdd292a6`, `6fb7e9d9`, `b2c50e88`, `6b2e94dd` (round 5, which added this +section) and the round-6 commit. The commits stay local on `development` until the increment is +accepted. **Review rounds.** Each round is a NEW independent complete cumulative review of `9902e0a0..HEAD`, every reviewer in its own worktree detached at the reviewed head: scrubber correctness, tests and causal @@ -9439,10 +9440,11 @@ stay local and Git-ignored. | 3 | `bdd292a6` | NONCLEAN | 17 | sequential rule application replaced by one merged, token-only masking pass | | 4 | `6fb7e9d9` | NONCLEAN (integration clean) | 19 | the literal query names in the merged pass; concatenated label runs and the nine-label bound; one standalone label rule for both labels; a bounded fixed point in both maskers; the secrets CI job allowance becomes 60 minutes | | 5 | `b2c50e88` | NONCLEAN | 20 (2 high) | core's round-4 authority extent erased the context its sequential rules read, so core now masks an ambiguous userinfo with exactly the query values its later rules mask whole; its split view masks URL userinfo before scheme labels; the LLM split scan retries a kept candidate glued across a line break; tested fixed-point helpers; untested pattern pieces gained discriminating tests | +| 6 | `6b2e94dd` | NONCLEAN | 21 (9 high) | round 5's value mirror was quadratic and still erased context, and its split view dropped the value mask; core now merges every rule's ranges on one text, as the scrubber does, reads a key or scheme label inside another value and a value through earlier masks, reads a split header label or a scheme label right after a credential, and scans PEM blocks linearly; the scrubber's nine-label bound also reads the raw text and its literal query names follow `#`; both fail-closed values are tested | **Author gates and controls.** Every corrected head passed `pnpm run ci` and the affected suites under `CI=true GITHUB_ACTIONS=true`, plus local differential, exhaustive and admitted-size harnesses that -found no new leak against the previous head. The secrets family is 110 controls / 330 phases / 482 +found no new leak against the previous head. The secrets family is 110 controls / 330 phases / 491 targets and witnesses, projecting 1,987 of the unchanged 2,000 retained files; [tools/secrets-controls/README.md](../../../tools/secrets-controls/README.md) records each round's control changes and, from round 5, the rule that pieces inside a mechanism's pattern are pinned by diff --git a/docs/standards/error-handling.md b/docs/standards/error-handling.md index 989deff9..9ff7f1e9 100644 --- a/docs/standards/error-handling.md +++ b/docs/standards/error-handling.md @@ -129,7 +129,7 @@ message, and the backstop masks a finite set of secret **shapes**, not arbitrary content. - **Shapes.** OpenAI/Anthropic `sk-…` and Google `AIza…` key prefixes; a finite list of literal - secret query names; the exact `api-key`, `x-api-key` and `x-goog-api-key` header names, also + secret query names, after `?`, `&` or `#` (an OAuth implicit-flow `#access_token=`); the exact `api-key`, `x-api-key` and `x-goog-api-key` header names, also in quoted, JSON-encoded and JSON-in-JSON (`\"x-api-key\":\"…\"`) objects; and `Bearer`/`Basic` authentication values. A header value may begin with up to eight (possibly quoted, escaped, punctuation-glued or LF/TAB-split) `Bearer`/`Basic` labels — a doubled @@ -169,8 +169,8 @@ content. any scheme; an empty password, or text glued after a masked one, is still masked. The authored-content detector's projection of `postgres://app:{{secrets.password}}@db` to `postgres://app:[REDACTED]@db` is therefore a fixed point. Core's journal projection applies the - same `?`/`#` predicate, but its rules run in sequence, so it masks an ambiguous userinfo together - with the query values its later rules would mask whole instead of through the authority's end + same `?`/`#` predicate and also merges its rules' ranges, but masks an ambiguous userinfo together + with the query values in it, each to its own end, rather than through the authority's end ([built-in tools](../reference/shared-core/built-in-tools.md#finite-diagnostic-credential-masking)). - **Split and normalized forms.** Every rule except the literal query names also probes the terminal-normalized text; a credential revealed only by normalization masks the **whole** @@ -189,7 +189,7 @@ content. after the last `@` of such an ambiguous authority unless it is host-shaped; a `Bearer`/`Basic` label that ends a line takes the next token with it; an opaque value after a label ends only at whitespace, so in compact JSON it masks to the end of the run; nine or more - concatenated labels, and a token that begins with `,`, `;` or `}` after a `Bearer`/`Basic` + concatenated labels, read in the raw text as well as the normalized one, and a token that begins with `,`, `;` or `}` after a `Bearer`/`Basic` label in a header value, blank the whole diagnostic. Past eight separate labels after a header name, and in a value of labels alone glued behind punctuation (`Bearer |Bearer`), the outcome depends on what precedes the header name: when its nearest non-whitespace character is a word @@ -208,7 +208,12 @@ content. an ordinary `connect to sftp://user@host` diagnostic. (A word that ends a line before `ftp://` is no such glue: an LF/TAB-split ftp credential after it blanks the diagnostic.) Malformed URLs (`https:/u:p@h`), a raw `/` inside a password, invisible characters inside a scheme and JSON - whose quotes are themselves escaped as `\u0022` are outside the finite shapes. + whose quotes are themselves escaped as `\u0022` are outside the finite shapes. In a JSON-encoded + diagnostic, an escape written as two characters (`\n`, `\t`, `\r`) and glued before a key prefix, + a header name or a label defeats that shape's word boundary, so the credential after it stays + visible (`"key rejected:\nsk-…"`); core's journal projection shares this limit. A header value + that begins with `,`, `;` or `}` and has no label before it is an empty token, so the text after + it stays visible. ## No silent catches diff --git a/packages/core/src/tools/bounding.test.ts b/packages/core/src/tools/bounding.test.ts index 897ea9ab..d74f42c7 100644 --- a/packages/core/src/tools/bounding.test.ts +++ b/packages/core/src/tools/bounding.test.ts @@ -9,8 +9,16 @@ import { redactSecretShapedValue, utf8ByteLength, } from './bounding.js'; +import { toFixedPoint } from './fixed-point.js'; import type { ToolHost, ToolResultLimits } from './types.js'; +// Observe the projection's bounded fixed point, retaining its actual implementation, so one case can +// make it fail and see which value the projection fails closed with. +vi.mock('./fixed-point.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, toFixedPoint: vi.fn(actual.toFixedPoint) }; +}); + const BIG: ToolResultLimits = { maxBytes: 50_000, maxLines: 2000 }; const TINY: ToolResultLimits = { maxBytes: 20, maxLines: 3 }; @@ -1473,3 +1481,265 @@ describe('PR95 review round 5: an ambiguous userinfo masked with exactly the val expect(output === 'Bearer [redacted]://h.example', 'SECRETS_B5_CORE_SCHEME_ORDER').toBe(true); }, 15_000); }); + +describe('PR95 review round 6: one merged pass of every core rule', () => { + const pem = [ + '-----BEGIN RSA PRIVATE KEY-----', + 'MIIBOgIBAAJBAKj34GkxFhD90vcNLYLInFEX6Ppy1tPf9Cnzj4p4WGeKLs1Pt8Qu', + '-----END RSA PRIVATE KEY-----', + ].join('\n'); + const projects = (marker: string, pairs: readonly (readonly [string, string, string])[]) => { + for (const [input, expected, secret] of pairs) { + const output = redactSecretShapedText(input); + expect(output, marker).toBe(expected); + expect(output.includes(secret), marker).toBe(false); + expect(redactSecretShapedText(output), marker).toBe(output); + } + }; + + it('keeps every credential and its context when core masks overlap', () => { + projects('SECRETS_B6_CORE_MERGE', [ + [ + 'https://a?b:c&password=xy@Bearer abcdefgh12345678', + 'https://[redacted] [redacted]', + 'abcdefgh12345678', + ], + [ + 'https://svc?password=abcd@c/Bearer abcdefgh12345678', + 'https://[redacted] [redacted]', + 'abcdefgh12345678', + ], + [`https://a?b:c&password=xy@${pem}`, 'https://[redacted]', 'MIIBOgIBAAJBAKj34'], + [ + `https://u:pw?x=${'a'.repeat(24)}@h;Bearer abcdefgh12345678`, + 'https://[redacted] [redacted]', + 'abcdefgh12345678', + ], + [ + 'postgres://app:pw?application_name=reporting-service-01@db;password="S3cr3tValue99"', + 'postgres://[redacted]', + 'S3cr3tValue99', + ], + [ + 'mysql://db?user=root&password=Adm1n@2024.prod/Bearer hunter2secretvalue', + 'mysql://[redacted] [redacted]', + 'hunter2secretvalue', + ], + [`redis://db#client_secret=G5oK47GKkhEp@2024.prod/${pem}`, 'redis://[redacted]', 'MIIBOg'], + ]); + }); + + it.each(['\n', '\t', 'word\t', 'DATABASE_URL:\n'])( + 'masks an ambiguous userinfo with its query values behind %j', + (prefix) => { + projects('SECRETS_B6_CORE_MERGE', [ + [`${prefix}https://gateway?password=P@ssw0rd123`, `${prefix}https://[redacted]`, 'ssw0rd'], + [ + `${prefix}mysql://db?user=root&password=Adm1n@2024.prod`, + `${prefix}mysql://[redacted]`, + '2024.prod', + ], + [ + `${prefix}see https://svc?token=abcdefghij@foo.example/x for details`, + `${prefix}see https://[redacted] for details`, + 'foo.example', + ], + [ + `${prefix}https://svc?sig=${'A'.repeat(24)}@tail.example`, + `${prefix}https://[redacted]`, + 'tail.example', + ], + ]); + }, + ); + + it('masks an ambiguous userinfo whose scheme is split by an LF', () => { + projects('SECRETS_B6_CORE_MERGE', [ + ['https:/\n/svc?password=Secret@tail99', 'https:/\n/[redacted]', 'tail99'], + ['https://\nsvc?password=Secret@tail99', 'https://[redacted]', 'tail99'], + ]); + }); + + it("reads a key inside another key's value", () => { + projects('SECRETS_B6_CORE_NESTED_KEY', [ + ['token=abcdef>password: hunter2hunter2', '[redacted]', 'hunter2hunter2'], + [ + 'https://user:pw?token=K9wR2v@db/x>password: hunter2hunter2', + 'https://[redacted]', + 'hunter2hunter2', + ], + [ + 'postgres://u:p?password=K9wR2v@db/path[api_key: SECRETVALUE99', + 'postgres://[redacted]', + 'SECRETVALUE99', + ], + [ + '{"msg":"http://app:pw?a=1&x=OGePIzn69aW2APSxEBy7@def.example\\nx-api-key: R2FIHKWkS0J2kZ0L35"}', + '{"msg":"http://[redacted]"}', + 'R2FIHKWkS0J2kZ0L35', + ], + ['x-api-key: "abc password="hunter2 secret value"', '[redacted]', 'secret value'], + ]); + }); + + it("reads a scheme label inside another label's token", () => { + projects('SECRETS_B6_CORE_NESTED_LABEL', [ + ['Bearer abcdefgh/Bearer hunter2secretvalue', 'Bearer [redacted] [redacted]', 'hunter2'], + ['token abcdefgh12/Basic hunter2secretvalue==', 'token [redacted] [redacted]', 'hunter2'], + ]); + }); + + it('reads a value through the masks before it, line breaks included', () => { + projects('SECRETS_B6_CORE_VALUE_THROUGH_MASK', [ + [ + '{"a":"x-api-key": "QL9uUu2jzxnM\txoxb-QWZDhKp63WKU\naccess_token="Qi3YmmwU37GH Qqu6LcyxbzW8"', + '{"a":"[redacted]', + 'QL9uUu2jzxnM', + ], + [ + `x-api-key: "v@${pem}client_secret="Q9w8EwQDxFP2 QmeNGkQBxtfm"`, + '[redacted]', + 'Q9w8EwQDxFP2', + ], + [ + 'token: "abc sk-proj-abcdefghij\nklmnopqrstuv" password: "secret value 1"', + '[redacted] [redacted]', + 'secret value', + ], + ]); + }); + + it('joins a split header label right after a credential or a PEM block', () => { + projects('SECRETS_B6_CORE_MASK_BOUNDARY', [ + [ + 'url: AKIA\nQRTBLS2XNANSABCDx-api-\tkey": "QzJsBjkrL2GS?access_token="QeKrgCE47mCW QW2Y8jaZkAPA"', + 'url: [redacted]', + 'QzJsBjkrL2GS', + ], + [`${pem}authori\nzation: QFzQnbCdq2TL`, '[redacted]', 'QFzQnbCdq2TL'], + ['AKIAQNNKD8CEOFHOABCDauthori\nzation:QNauurwS3vLp', '[redacted]', 'QNauurwS3vLp'], + ]); + }); + + it('reads a scheme label or a credential glued to the end of another credential', () => { + projects('SECRETS_B6_CORE_MASK_BOUNDARY', [ + [ + 'x\n\tclient_secret=QRYyV2UYKhnz!AKIAQPJ53XE28CC4A\nBCDtoken QNZd4i2J6YY5', + 'x\n\t[redacted] [redacted]', + 'QNZd4i2J6YY5', + ], + ['AKIAQPJ53XE28CC4ABCDtoken QNZd4i2J6YY5', '[redacted]token [redacted]', 'QNZd4i2J6YY5'], + [ + 'client_secret=QRYyV2UYKhnz!AKIAQPJ53XE28CC4ABCDtoken QNZd4i2J6YY5', + '[redacted] [redacted]', + 'QNZd4i2J6YY5', + ], + ['AKIAQPJ53XE28CC4ABCDsk-proj-abcdefghijklmnopqrstuv', '[redacted][redacted]', 'ghijklmnop'], + [ + 'AKIAQPJ53XE28CC4A\nBCDsk-proj-abcdefghijklmnopqrstuv', + '[redacted][redacted]', + 'ghijklmnop', + ], + ]); + }); + + it('keeps the host of a URL whose userinfo names a secret', () => { + const userinfo = (value: unknown) => expect(value, 'SECRETS_B6_CORE_USERINFO_ZONE'); + for (const [input, expected] of [ + [ + 'https://x-access-token:ghp_abcdefghijklmnopqrstuvwxyz0123@github.com/org/repo.git', + 'https://[redacted]@github.com/org/repo.git', + ], + [ + 'https://token:abcdef@h.example/x?password=hunter2hunter2', + 'https://[redacted]@h.example/x?[redacted]', + ], + [`https://a&sig=${'X'.repeat(28)}:pw@h.example/`, 'https://[redacted]@h.example/'], + // Before a raw ?/#, a `user:password` owns its text; the query after it stays readable. + ['https://token:pa?ss@h.example/repo', 'https://[redacted]@h.example/repo'], + ['https://token:abc?x=1@host/path?y=2', 'https://[redacted]@host/path?y=2'], + ] as const) { + userinfo(redactSecretShapedText(input)).toBe(expected); + } + }); + + it('reads every key of an ambiguous userinfo that names no user', () => { + projects('SECRETS_B6_CORE_USERINFO_COLON', [ + ['https://password=Secret1?x@t123456', 'https://[redacted]', 't123456'], + [`https://a&sig=${'X'.repeat(28)}?y@tail99`, 'https://[redacted]', 'tail99'], + ['https://svc?a=1;password=P@ssw0rd123', 'https://[redacted]', 'ssw0rd123'], + ['https://svc?password:P@ssw0rd99', 'https://[redacted]', 'ssw0rd99'], + ['https://svc?x!password=Secr@t123456', 'https://[redacted]', 't123456'], + ['https://svc?a=b+password=Secr@t123456', 'https://[redacted]', 't123456'], + [`https://svc?${'n'.repeat(70)}.password=Secr@t123456`, 'https://[redacted]', 't123456'], + ]); + }); + + it('projects a split URL into the summary and the journal value alike', async () => { + const result = await boundForModel('see\nhttps://gateway?password=P@ssw0rd123 now', BIG, {}); + expect(result.summary, 'SECRETS_B6_CORE_MERGE').toBe('see https://[redacted] now'); + expect( + redactSecretShapedValue({ url: 'x\thttps://gateway?password=P@ssw0rd123' }), + 'SECRETS_B6_CORE_MERGE', + ).toEqual({ url: 'x\thttps://[redacted]' }); + }); + + it('keeps many query assignments in an ambiguous userinfo linear at admitted size', () => { + const size = 8 * 1024 * 1024 - 256; + const run = (prefix: string, unit: string, suffix: string) => + prefix + + unit.repeat(Math.floor((size - prefix.length - suffix.length) / unit.length)) + + suffix; + for (const [input, expected] of [ + [run('https://x', '?password=', '@h'), 'https://[redacted]'], + [run('https://x', '?a=', '@h'), 'https://[redacted]'], + [run('https://svc?', '#password=x', '@h'), 'https://[redacted]'], + [run('see https://svc', '?token=a@b', ''), 'see https://[redacted]'], + [run('redis://app:pw', '?client_secret=x', '@db'), 'redis://[redacted]'], + ] as const) { + let output = ''; + expect(() => { + output = redactSecretShapedText(input); + }, 'SECRETS_B6_CORE_MERGE').not.toThrow(); + expect(output === expected, 'SECRETS_B6_CORE_MERGE').toBe(true); + } + }, 15_000); + + it('keeps nested values, labels and open PEM blocks linear at admitted size', () => { + const size = 8 * 1024 * 1024 - 256; + const count = (unit: string) => Math.floor(size / unit.length); + const keys = 'token=ab>'.repeat(count('token=ab>')); + expect(redactSecretShapedText(keys) === '[redacted]', 'SECRETS_B6_CORE_NESTED_KEY').toBe(true); + const labels = count('Bearer abcdefgh/'); + expect( + redactSecretShapedText('Bearer abcdefgh/'.repeat(labels)) === + `Bearer${' [redacted]'.repeat(labels)}`, + 'SECRETS_B6_CORE_NESTED_LABEL', + ).toBe(true); + // Only the BEGIN marker whose body reaches the far END within the body bound opens a block; the + // markers before it stay, and each of them reuses one END search instead of scanning to it again. + const marker = '-----BEGIN PRIVATE KEY-----\n'; + const open = marker.repeat(count(marker) - 1); + const first = + Math.ceil((open.length - 20_000 - (marker.length - 1)) / marker.length) * marker.length; + expect( + redactSecretShapedText(`password=hunter2secret ${open}-----END PRIVATE KEY-----`) === + `[redacted] ${open.slice(0, first)}[redacted]`, + 'SECRETS_B6_CORE_MERGE', + ).toBe(true); + }, 15_000); + + it('masks a PEM block whose body reaches exactly the body bound', () => { + const begin = '-----BEGIN PRIVATE KEY-----'; + const end = '-----END PRIVATE KEY-----'; + const bounded = (value: unknown) => expect(value, 'SECRETS_B6_CORE_MERGE'); + bounded(redactSecretShapedText(`${begin}${'A'.repeat(20_000)}${end}`)).toBe('[redacted]'); + const longer = `${begin}${'A'.repeat(20_001)}${end}`; + bounded(redactSecretShapedText(longer) === longer).toBe(true); + }); + + it('fails closed with its own mask when the projection does not settle', () => { + vi.mocked(toFixedPoint).mockImplementationOnce((_pass, _text, failed) => failed); + expect(redactSecretShapedText('public text'), 'SECRETS_B6_CORE_FAIL_CLOSED').toBe('[redacted]'); + }); +}); diff --git a/packages/core/src/tools/bounding.ts b/packages/core/src/tools/bounding.ts index f6e8d2fd..02973ab7 100644 --- a/packages/core/src/tools/bounding.ts +++ b/packages/core/src/tools/bounding.ts @@ -163,34 +163,50 @@ const JOINED_STANDALONE_CREDENTIAL = new RegExp(STANDALONE_CREDENTIAL.source.sli const SCHEME_CREDENTIAL = /\b(b[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|b[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c|t[\n\t]*o[\n\t]*k[\n\t]*e[\n\t]*n)\s+(?=[\w.~+/-]{8})[\w.~+/-]+={0,2}/gi; -// A secret-ish key (bounded wrappers keep this ReDoS-safe) + `=`/`:` + its value. The optional `["']?` -// BEFORE the separator catches the JSON `"access_token":"…"` shape (an OAuth/egress response body). The -// value has QUOTED and UNQUOTED branches. A quoted value ends at the first UNESCAPED matching quote, -// including JSON's escaped quotes/backslashes; otherwise a valid password such as `a\"secret-tail` -// left its tail exposed. The two alternatives inside each repetition are disjoint: a backslash -// consumes exactly the following non-newline character, while the ordinary class excludes it. -// No ambiguous nested repetition/backref is needed. Unquoted values retain their delimiter and -// minimum-length policy. Kept a single analyzable LITERAL (not composed via `new RegExp`) so Sonar's static -// super-linear-runtime (S5852) check still covers this security-critical pattern; its keyword-alternation -// breadth is deliberate (and `apikey` is dropped — `api[_-]?key` already subsumes it). -const SECRET_KEY_VALUE = - /\b[\w-]{0,32}(?:password|passwd|secret|token|api[_-]?key|authorization|access[_-]?key|private[_-]?key|client[_-]?secret)[\w-]{0,16}["']?\s*[=:]\s*(?:"(?:[^"\\\r\n]|\\[^\r\n])*"|'(?:[^'\\\r\n]|\\[^\r\n])*'|(?=[^\s"',;&]{6})[^\s"',;&]+)/gi; -/** The same rule anchored at one position, for a query value inside an ambiguous userinfo. */ -const SECRET_KEY_VALUE_AT = new RegExp(SECRET_KEY_VALUE.source, 'iy'); -/** A query name the secret key=value rule reads, from the word boundary where that rule would start. */ -const SECRET_QUERY_NAME = - /\b[\w-]{0,32}(?:password|passwd|secret|token|api[_-]?key|authorization|access[_-]?key|private[_-]?key|client[_-]?secret)[\w-]{0,16}$/i; +// A secret-ish key (bounded wrappers keep this ReDoS-safe) + `=`/`:`. The optional `["']?` BEFORE the +// separator catches the JSON `"access_token":"…"` shape (an OAuth/egress response body). Kept an +// analyzable LITERAL (not composed via `new RegExp`) so Sonar's static super-linear-runtime (S5852) +// check still covers this security-critical pattern; its keyword-alternation breadth is deliberate +// (and `apikey` is dropped — `api[_-]?key` already subsumes it). The value is read after it, in +// `visitSecretKeyValues`. +const SECRET_KEY = + /\b[\w-]{0,32}(?:password|passwd|secret|token|api[_-]?key|authorization|access[_-]?key|private[_-]?key|client[_-]?secret)[\w-]{0,16}["']?\s*[=:]\s*/gi; +// A QUOTED value ends at the first UNESCAPED matching quote, including JSON's escaped quotes/backslashes; +// otherwise a valid password such as `a\"secret-tail` left its tail exposed. The two alternatives inside +// each repetition are disjoint: a backslash consumes exactly the following non-newline character, while +// the ordinary class excludes it. No ambiguous nested repetition/backref is needed. +const DOUBLE_QUOTED_VALUE = /"(?:[^"\\\r\n]|\\[^\r\n])*"/y; +const SINGLE_QUOTED_VALUE = /'(?:[^'\\\r\n]|\\[^\r\n])*'/y; /** - * Masks one recognized span. `splitFrom`, when given, is where the credential's identifying prefix - * starts (a URL's scheme): a removed LF/TAB anywhere from there on splits the credential, so a - * separator inside `scheme://` counts even though only the userinfo is replaced. + * One recognized span of the projection's input and the text that replaces it. Every rule reports + * its spans on the same text, and overlapping spans merge before anything is replaced + * ({@link replaceRanges}), so no rule's mask can erase the key name, scheme label, PEM marker or URL + * scheme another rule reads. */ +interface SecretRange { + readonly start: number; + readonly end: number; + readonly replacement: string; +} + +/** + * A URL userinfo's context. `schemeStart` is where its identifying prefix starts: a removed LF/TAB + * anywhere from there on splits the credential, so a separator inside `scheme://` counts even though + * only the userinfo is replaced. Up to `ownsUntil` the userinfo owns its text: a key or query + * parameter there names a username or password, not a credential of its own. + */ +interface UrlUserinfo { + readonly schemeStart: number; + readonly ownsUntil: number; +} + +/** Reports one recognized span; `url` is set for a URL's userinfo. */ type SecretSpanVisitor = ( start: number, end: number, replacement: string, - splitFrom?: number, + url?: UrlUserinfo, ) => void; /** @@ -273,18 +289,6 @@ function visitJoinedStandaloneCredentials( } } -function redactStandaloneCredentials(text: string): string { - const parts: string[] = []; - let copied = 0; - visitJoinedStandaloneCredentials(text, text, (start, end, replacement) => { - parts.push(text.slice(copied, start), replacement); - copied = end; - }); - if (parts.length === 0) return text; - parts.push(text.slice(copied)); - return parts.join(''); -} - /** * The finite header-label literal is the sole joined-header authority. Check ORIGINAL boundaries * before matching, so an embedded x-api-key/x-goog-api-key cannot consume an inner api-key start. @@ -319,25 +323,25 @@ function visitJoinedHeaderLabels(original: string, joined: string, mask: SecretS } /** - * LF/TAB may split finite credential shapes. Map recognized spans back onto the original text; - * ordinary spaces remain boundaries and public text is never globally normalized for identity. - * Token-character continuations after LF/TAB are intentionally part of a recognized credential. - * The two cursors advance monotonically, with no per-character offset array or recursive matcher. + * LF/TAB may split finite credential shapes. Map the spans recognized in the joined view back onto + * the original text and report the ones a removed separator actually splits; ordinary spaces remain + * boundaries and public text is never globally normalized for identity. Token-character + * continuations after LF/TAB are intentionally part of a recognized credential. The two cursors + * advance monotonically, with no per-character offset array or recursive matcher. */ -function redactJoinedCredentialSpans( +function visitSplitSpans( text: string, visit: (joined: string, mask: SecretSpanVisitor) => void, + report: SecretSpanVisitor, boundary?: RegExp, // URL userinfo owns a separator at either edge: one right after the scheme or before the @ splits it. ownsEdges = false, -): string { - if (!/[\n\t]/.test(text)) return text; +): void { + if (!/[\n\t]/.test(text)) return; const joined = text.replace(/[\n\t]+/g, ''); - const parts: string[] = []; const separators = /[\n\t]+/g; let next = separators.exec(text); let removed = 0; - let copied = 0; function advance(target: number, includeAtBoundary = false): number { while ( next !== null && @@ -348,40 +352,121 @@ function redactJoinedCredentialSpans( } return target + removed; } - visit(joined, (start, end, replacement, splitFrom) => { - const originalSplitFrom = splitFrom === undefined ? undefined : advance(splitFrom); + visit(joined, (start, end, replacement, url) => { + // A separator right before a URL's scheme is outside the URL, so it splits nothing. + const schemeStart = url === undefined ? undefined : advance(url.schemeStart, true); const originalStart = advance(start, !ownsEdges); + const ownsUntil = url === undefined ? undefined : advance(url.ownsUntil, ownsEdges); const originalEnd = advance(end, ownsEdges); if (boundary?.test(text[originalStart - 1] ?? '')) return; - if (!/[\n\t]/.test(text.slice(originalSplitFrom ?? originalStart, originalEnd))) return; - parts.push(text.slice(copied, originalStart), replacement); - copied = originalEnd; + if (!/[\n\t]/.test(text.slice(schemeStart ?? originalStart, originalEnd))) return; + report( + originalStart, + originalEnd, + replacement, + schemeStart === undefined || ownsUntil === undefined ? undefined : { schemeStart, ownsUntil }, + ); }); - if (parts.length === 0) return text; - parts.push(text.slice(copied)); - return parts.join(''); } -function redactSplitCredentials(text: string): string { - if (!/[\n\t]/.test(text)) return text; - const urls = redactJoinedCredentialSpans(text, visitUrlUserinfo, undefined, true); - const keys = redactJoinedCredentialSpans( - urls, +/** LF/TAB-split URL userinfo and standalone credentials (`visitSplitSpans`). */ +function visitSplitCredentials(text: string, report: SecretSpanVisitor): void { + visitSplitSpans(text, visitUrlUserinfo, report, undefined, true); + visitSplitSpans( + text, (joined, mask) => { - visitJoinedStandaloneCredentials(urls, joined, mask); + visitJoinedStandaloneCredentials(text, joined, mask); }, + report, /\w/, ); - // A scheme label may take a URL scheme as its token, and a URL without its scheme hides its - // userinfo from every later rule, so the userinfo is masked first, as in the main chain. - const schemes = redactUrlUserinfo(keys).replace(SCHEME_CREDENTIAL, '$1 [redacted]'); - return redactJoinedCredentialSpans( - schemes, - (joined, mask) => { - visitJoinedHeaderLabels(schemes, joined, mask); +} + +/** + * The text as a rule reads it right after the given credentials: each one is filled like an applied + * mask, so a label or credential glued to its end still starts at a word boundary. + */ +function afterMasks(text: string, ranges: readonly SecretRange[]): string { + return fillRanges(text, ranges); +} + +/** + * Join every LF/TAB-split header label, so the key=value rule reads it like an unsplit one. A label + * starts at a word boundary, and a credential or a PEM block right before it is one (`afterMasks`). + */ +function joinSplitHeaderLabels(text: string): string { + if (!/[\n\t]/.test(text)) return text; + const masks: SecretRange[] = []; + const mask: SecretSpanVisitor = (start, end, replacement) => { + masks.push({ start, end, replacement }); + }; + visitSplitCredentials(text, mask); + visitPemBlocks(text, mask); + visitJoinedStandaloneCredentials(text, text, mask); + const boundaries = afterMasks(text, masks); + const labels: SecretRange[] = []; + visitSplitSpans( + boundaries, + (joined, label) => { + visitJoinedHeaderLabels(boundaries, joined, label); + }, + (start, end, replacement) => { + labels.push({ start, end, replacement }); }, /[A-Za-z0-9_-]/, ); + return replaceRanges(text, labels); +} + +/** + * Replace each run of overlapping ranges once, with the replacement of the range that starts it. + * Ranges that only touch stay separate runs, so a key=value mask and the scheme label's token after + * it read `[redacted] [redacted]`. + */ +function replaceRanges(text: string, ranges: readonly SecretRange[]): string { + const ordered = [...ranges].sort((left, right) => left.start - right.start); + const parts: string[] = []; + let copied = 0; + let run: { start: number; end: number; replacement: string } | undefined; + for (const range of ordered) { + if (run !== undefined && range.start < run.end) { + run.end = Math.max(run.end, range.end); + continue; + } + if (run !== undefined) { + parts.push(text.slice(copied, run.start), run.replacement); + copied = run.end; + } + run = { start: range.start, end: range.end, replacement: range.replacement }; + } + if (run === undefined) return text; + parts.push(text.slice(copied, run.start), run.replacement, text.slice(run.end)); + return parts.join(''); +} + +/** + * The same text with each run of overlapping ranges replaced by a run of the same length that reads + * like a mask (`[xx…x]`): no line break, quote, delimiter or key inside, and a word boundary at either + * end. Offsets stay valid, so a rule that reads it reports ranges of the original text. + */ +function fillRanges(text: string, ranges: readonly SecretRange[]): string { + const runs: { start: number; end: number }[] = []; + for (const range of [...ranges].sort((left, right) => left.start - right.start)) { + if (range.end <= range.start) continue; + const last = runs.at(-1); + if (last !== undefined && range.start < last.end) last.end = Math.max(last.end, range.end); + else runs.push({ start: range.start, end: range.end }); + } + if (runs.length === 0) return text; + const parts: string[] = []; + let copied = 0; + for (const run of runs) { + const length = run.end - run.start; + parts.push(text.slice(copied, run.start), length < 2 ? '[' : `[${'x'.repeat(length - 2)}]`); + copied = run.end; + } + parts.push(text.slice(copied)); + return parts.join(''); } /** @@ -403,33 +488,55 @@ export function redactSecretShapedText(text: string): string { } function redactSecretShapedOnce(text: string): string { - const projected = redactOpaqueQueryValues(redactUrlUserinfo(redactSplitCredentials(text))) - // A PEM private-key block (multi-line, space-separated markers the `private_key` key-pattern can't see). - // The body span is bounded (`{0,20000}?`, lazy) so an unterminated block can't drive an unbounded scan. - .replace( - /-----BEGIN [A-Z0-9 ]{0,40}PRIVATE KEY-----[\s\S]{0,20000}?-----END [A-Z0-9 ]{0,40}PRIVATE KEY-----/g, - '[redacted]', - ) - // An `Authorization`-style scheme + its token: `Bearer ` / `Basic ` / `Token `. The token class - // `[\w.~+/-]` is base64url + padding (`\w` folds `[A-Za-z0-9_]`). - // For every open-ended token minimum below, a bounded lookahead enforces the same floor before - // one simple greedy run. V8 can exhaust its RegExp stack on an admitted-scale `{N,}` run; the - // equivalent `(?=class{N})class+` form keeps the whole value instead of clipping a credential. - .replace(SCHEME_CREDENTIAL, '$1 [redacted]') - // A secret-ish key and its value (`SECRET_KEY_VALUE`). - .replace(SECRET_KEY_VALUE, '[redacted]'); + const source = joinSplitHeaderLabels(text); + return replaceRanges(source, secretRanges(source)); +} + +/** Every span the projection masks in `text`; each rule is described at its visitor. */ +function secretRanges(text: string): readonly SecretRange[] { + const ranges: SecretRange[] = []; + // Ranges merge instead of erasing context, yet some reads see the text as already masked: a key or + // parameter in a URL userinfo's own text (`UrlUserinfo.ownsUntil`) is that userinfo's, a key's + // value runs through every span an earlier rule masks, line breaks included, and a scheme label + // may start right after a credential, as each would with the masks applied. + const owned: SecretRange[] = []; + const masked: SecretRange[] = []; + const report: SecretSpanVisitor = (start, end, replacement, url) => { + const range = { start, end, replacement }; + ranges.push(range); + masked.push(range); + if (url !== undefined) owned.push({ start, end: url.ownsUntil, replacement }); + }; + const record: SecretSpanVisitor = (start, end, replacement) => { + ranges.push({ start, end, replacement }); + }; + visitSplitCredentials(text, report); + visitUrlUserinfo(text, report); + const names = fillRanges(text, owned); + visitOpaqueQueryValues(names, text, report); + visitPemBlocks(text, report); + // No value runs through a scheme label's token: `authorization: Bearer ` keeps both masks. + visitSchemeCredentials(text, record); + visitSecretKeyValues(names, fillRanges(text, masked), record); // Well-known standalone credential shapes (OpenAI, Stripe, AWS incl. STS ASIA/ABIA, GitHub token + PAT, // GitLab PAT, Slack, Google API + OAuth, HuggingFace, npm, JWT). Kept as ONE alternation ON PURPOSE: a - // single leftmost-longest scan is required so two ADJACENT / EMBEDDED credential shapes (e.g. a JWT whose - // signature segment contains an `sk-` run, or a `xox…-glpat-…` run) are covered by the outer greedy match - // as a single span. A per-family MULTI-PASS split is NOT equivalent — an earlier pass inserting `[redacted]` - // (a non-word `[`) truncates a later family's greedy match and can leave a trailing secret-shaped substring - // EXPOSED (a real redaction regression, pinned in bounding.test.ts). The canonical fragments therefore - // construct one complete alternation, with a separate bounded prefix-absence preflight. Failed probes - // use the bounded preview before the full greedy matcher; overlap and admitted-size tests pin that path. + // single leftmost-longest scan covers two ADJACENT / EMBEDDED credential shapes (e.g. a JWT whose + // signature segment contains an `sk-` run, or a `xox…-glpat-…` run) with the outer greedy match as a + // single span (pinned in bounding.test.ts). The canonical fragments therefore construct one complete + // alternation, with a separate bounded prefix-absence preflight. Failed probes use the bounded + // preview before the full greedy matcher; overlap and admitted-size tests pin that path. // `\w` / `[\w-]` fold only the classes that are EXACTLY `[A-Za-z0-9_]` / `[A-Za-z0-9_-]`; // the tighter `[A-Za-z0-9]` / `[A-Za-z0-9-]` families keep their narrower class (no `_`). - return redactStandaloneCredentials(projected); + const standalone: SecretRange[] = []; + visitJoinedStandaloneCredentials(text, text, (start, end, replacement) => { + record(start, end, replacement); + standalone.push({ start, end, replacement }); + }); + // A scheme label glued to the end of another credential starts one of its own, even where a key's + // value runs on over it: labels are read again with every mask filled but key values and scheme + // tokens (`afterMasks`). Anything else glued to a mask is read by the projection's next pass. + visitSchemeCredentials(afterMasks(text, [...masked, ...standalone]), record); + return ranges; } /** @@ -438,25 +545,105 @@ function redactSecretShapedOnce(text: string): string { * mask the whole value: a fixed-length prefix leaves a permanent credential equality oracle. * Scan delimiters separately instead of matching a multi-megabyte repeated value in V8's * RegExp stack. Both cursors advance; short unrecognized values add only bounded overlap. + * Parameters are read in `names`, which hides each URL userinfo's own text (a parameter there is the + * userinfo's); a value is measured in `text`, so a mask inside it neither ends nor extends it. */ -function redactOpaqueQueryValues(text: string): string { - if (!text.includes('?') && !text.includes('&')) return text; +function visitOpaqueQueryValues(names: string, text: string, mask: SecretSpanVisitor): void { + if (!names.includes('?') && !names.includes('&')) return; const prefix = /[?&][\w.-]{1,64}=/g; const delimiter = /[\s&#"',}\]]/g; - const parts: string[] = []; - let cursor = 0; - for (let match = prefix.exec(text); match !== null; match = prefix.exec(text)) { + for (let match = prefix.exec(names); match !== null; match = prefix.exec(names)) { const valueStart = prefix.lastIndex; delimiter.lastIndex = valueStart; const valueEnd = delimiter.exec(text)?.index ?? text.length; if (valueEnd - valueStart < 24) continue; - parts.push(text.slice(cursor, valueStart), '[redacted]'); - cursor = valueEnd; + mask(valueStart, valueEnd, '[redacted]'); prefix.lastIndex = valueEnd; } - if (parts.length === 0) return text; - parts.push(text.slice(cursor)); - return parts.join(''); +} + +/** A PEM block's body is bounded, so an unterminated block cannot drive an unbounded scan. */ +const PEM_BODY_MAX = 20_000; + +/** + * A PEM private-key block (multi-line, space-separated markers the `private_key` key-pattern can't see): + * a BEGIN marker and the first END marker that starts at most PEM_BODY_MAX characters after it, which + * is what one regex with a lazy bounded body finds. That regex rescanned the whole bounded body after + * every unterminated BEGIN marker; here one END search serves every BEGIN marker before the END it + * found, so the scan stays linear. + */ +function visitPemBlocks(text: string, mask: SecretSpanVisitor): void { + if (!text.includes('-----BEGIN ')) return; + const begin = /-----BEGIN [A-Z0-9 ]{0,40}PRIVATE KEY-----/g; + const end = /-----END [A-Z0-9 ]{0,40}PRIVATE KEY-----/g; + let found: RegExpExecArray | null | undefined; + for (let marker = begin.exec(text); marker !== null; marker = begin.exec(text)) { + const body = begin.lastIndex; + if (found === undefined || (found !== null && found.index < body)) { + end.lastIndex = body; + found = end.exec(text); + } + if (found === null) return; + if (found.index - body > PEM_BODY_MAX) { + // Like the regex, try the next start: a later BEGIN marker may be close enough to this END. + begin.lastIndex = marker.index + 1; + continue; + } + const blockEnd = found.index + found[0].length; + mask(marker.index, blockEnd, '[redacted]'); + begin.lastIndex = blockEnd; + } +} + +/** + * An `Authorization`-style scheme + its token: `Bearer ` / `Basic ` / `Token `. The token class + * `[\w.~+/-]` is base64url + padding (`\w` folds `[A-Za-z0-9_]`). The range starts after the label, so + * the label stays and one space separates it from the mask: `Bearer [redacted]`. The scan resumes + * after the label rather than after the token: a label inside the token (`Bearer a/Bearer `) starts + * a credential that reaches past this one. + * For every open-ended token minimum below, a bounded lookahead enforces the same floor before + * one simple greedy run. V8 can exhaust its RegExp stack on an admitted-scale `{N,}` run; the + * equivalent `(?=class{N})class+` form keeps the whole value instead of clipping a credential. + */ +function visitSchemeCredentials(text: string, mask: SecretSpanVisitor): void { + const scheme = new RegExp(SCHEME_CREDENTIAL); + for (let match = scheme.exec(text); match !== null; match = scheme.exec(text)) { + const labelEnd = match.index + (match[1]?.length ?? 0); + mask(labelEnd, scheme.lastIndex, ' [redacted]'); + scheme.lastIndex = labelEnd; + } +} + +/** + * A secret-ish key (`SECRET_KEY`) and its value: a whole quoted string, or an unquoted run of at least + * six characters up to whitespace, a quote, `,`, `;` or `&`. Keys are read in `keys`, which hides each + * URL userinfo's own text: a key there names its username or password + * (`https://x-access-token:@github.com/o/r`), and its value would run on through the public host and + * path. Values are read in `values`, where every earlier mask is one run, as an applied mask would be. + * The key scan resumes at each value rather than after it, so a key inside a value is read too: + * `token=a>password: ` has a second value past the first one's end. One delimiter scan serves + * every unquoted value, and a nested quoted value can only open at the closing quote of the one around + * it, so the scan stays linear. + */ +function visitSecretKeyValues(keys: string, values: string, mask: SecretSpanVisitor): void { + const key = new RegExp(SECRET_KEY); + const delimiter = /[\s"',;&]/g; + let delimiterAt = -1; + for (let match = key.exec(keys); match !== null; match = key.exec(keys)) { + const valueStart = key.lastIndex; + const quote = values[valueStart]; + if (quote === '"' || quote === "'") { + const quoted = quote === '"' ? DOUBLE_QUOTED_VALUE : SINGLE_QUOTED_VALUE; + quoted.lastIndex = valueStart; + if (quoted.test(values)) mask(match.index, quoted.lastIndex, '[redacted]'); + continue; + } + if (delimiterAt < valueStart) { + delimiter.lastIndex = valueStart; + delimiterAt = delimiter.exec(values)?.index ?? values.length; + } + if (delimiterAt - valueStart >= 6) mask(match.index, delimiterAt, '[redacted]'); + } } /** A host-shaped prefix: a dotted name (or IPv4), `localhost` or a bracketed IP literal, with an optional port. */ @@ -474,9 +661,10 @@ const HOST_SHAPED_MAX_LENGTH = 260; * begins public query/fragment text only when the text before it is host-shaped **and** an * `=` assignment follows it; anything else may be a malformed username or password. This is the * same predicate as the LLM diagnostic backstop (`packages/llm/src/scrub-secrets.ts`). For such an - * ambiguous userinfo the backstop masks through the authority's end, while this projection, whose rules - * run in sequence, masks it with the query values its later rules would mask whole (see - * `laterQueryValueEnd`). + * ambiguous userinfo the backstop masks through the authority's end unless a host-shaped name follows + * its last `@`. This projection masks the userinfo and reads the keys and parameters in it from the + * raw ?/# on, or all of them when no `user:password` comes first; their values keep their own extent + * and the rules' ranges merge (`https://gateway?password=P@ss.word1` projects to `https://[redacted]`). * The projection differs from that backstop by design: its authority also ends at `"`, `'`, `,`, * `}` and `]`, so userinfo containing one of them (an already-masked `[REDACTED]` password among * them) is left as it is; it recognizes any scheme of up to 32 characters rather than a finite list; @@ -490,36 +678,7 @@ function isPublicQueryContact(userinfo: string, query: number): boolean { return HOST_SHAPED_AUTHORITY.test(userinfo.slice(0, query)) && userinfo.includes('=', query + 1); } -/** A query assignment a later rule may read: a name and `=` after a raw `?`, `&` or `#`. */ -const QUERY_ASSIGNMENT = /[?&#]([\w.-]{1,64})=/g; - -/** - * Where core's later rules would end their masks of the query values that start inside an ambiguous - * userinfo (`from`..`to`), or -1: the opaque-query rule takes a value of 24 or more characters after - * `?`/`&`, and the secret key=value rule a secret-ish name's value. Masking the userinfo first would - * remove the names those rules read, so its mask runs to the same end and never farther, leaving - * every later credential's context in place. - */ -function laterQueryValueEnd(text: string, from: number, to: number): number { - let end = -1; - const opaqueEnd = /[\s&#"',}\]]/g; - for (const match of text.slice(from, to).matchAll(QUERY_ASSIGNMENT)) { - const valueStart = from + match.index + match[0].length; - if (match[0][0] !== '#') { - opaqueEnd.lastIndex = valueStart; - const valueEnd = opaqueEnd.exec(text)?.index ?? text.length; - if (valueEnd - valueStart >= 24) end = Math.max(end, valueEnd); - } - // The key=value rule may start at any word boundary inside the name, before its keyword. - const keyAt = (match[1] ?? '').search(SECRET_QUERY_NAME); - if (keyAt < 0) continue; - SECRET_KEY_VALUE_AT.lastIndex = from + match.index + 1 + keyAt; - if (SECRET_KEY_VALUE_AT.test(text)) end = Math.max(end, SECRET_KEY_VALUE_AT.lastIndex); - } - return end; -} - -function visitUrlUserinfo(text: string, mask: SecretSpanVisitor, extend = false): void { +function visitUrlUserinfo(text: string, mask: SecretSpanVisitor): void { if (!text.includes('://')) return; const scheme = /[a-z][\w+.-]{0,31}:\/\//gi; const delimiter = /[/\s"',}\]]/g; @@ -540,35 +699,19 @@ function visitUrlUserinfo(text: string, mask: SecretSpanVisitor, extend = false) if (query >= 0 && isPublicQueryContact(userinfo, query)) continue; const hasPassword = colon >= 0 && colon < userinfo.length - 1; if (!hasPassword && userinfo.length < 16) continue; - let maskEnd = start + at; - // A raw ?/# leaves the authority ambiguous: the userinfo may also be the query a later rule reads - // (`https://svc?password=P@ss`). Its mask also covers each value that rule would mask whole. The - // joined view (no `extend`) masks the userinfo only, so no value runs across a removed LF/TAB. - if (extend && query >= 0) { - maskEnd = Math.max(maskEnd, laterQueryValueEnd(text, start + query, start + at)); - } - mask(start, maskEnd, '[redacted]', match.index); - // Skip the masked span without hiding a later scheme in the public host/tail. - scheme.lastIndex = Math.max(maskEnd, start + at + 1); + // A raw ?/# leaves the authority ambiguous: from there on the userinfo may also be a query + // another rule reads (`https://gateway?password=P@ss.word1`). A `user:password` before it owns + // its text; with no colon there, nothing marks a username, so the userinfo owns none of it. + const ownsUntil = query < 0 ? at : colon >= 0 && colon < query ? query : 0; + mask(start, start + at, '[redacted]', { + schemeStart: match.index, + ownsUntil: start + ownsUntil, + }); + // Skip the recognized userinfo without hiding a later scheme in the public host/tail. + scheme.lastIndex = start + at + 1; } } -function redactUrlUserinfo(text: string): string { - const parts: string[] = []; - let cursor = 0; - visitUrlUserinfo( - text, - (start, end, replacement) => { - parts.push(text.slice(cursor, start), replacement); - cursor = end; - }, - true, - ); - if (parts.length === 0) return text; - parts.push(text.slice(cursor)); - return parts.join(''); -} - /** * Walk an arbitrary value applying {@link redactSecretShapedText} to every string, leaving structure intact — * the toolInput twin of the summary scrub. Applied to the `agent:tool_call.toolInput` event field so a @@ -805,7 +948,8 @@ function toText(result: unknown): string { function makeSummary(text: string): string { // Recognize LF/TAB-split shapes in the complete materialized text before one-line normalization. Clip only after recognition: a closing quote or URL // authority separator can be beyond the preview. Raw model/spill text stays unchanged. - const oneLine = redactSecretShapedText(redactSplitCredentials(text).replace(/\s+/g, ' ').trim()); + const recognized = /[\n\t]/.test(text) ? redactSecretShapedOnce(text) : text; + const oneLine = redactSecretShapedText(recognized.replace(/\s+/g, ' ').trim()); return oneLine.length <= SUMMARY_MAX ? oneLine : `${oneLine.slice(0, SUMMARY_MAX)}…`; } diff --git a/packages/llm/src/scrub-secrets.test.ts b/packages/llm/src/scrub-secrets.test.ts index ee7a6b5c..6447619b 100644 --- a/packages/llm/src/scrub-secrets.test.ts +++ b/packages/llm/src/scrub-secrets.test.ts @@ -1,9 +1,17 @@ -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; +import { toFixedPoint } from './fixed-point.js'; import { scrubSecrets as rootScrubSecrets } from './index.js'; import { makeLlmError, scrubSecrets as errorScrubSecrets } from './llm-error.js'; import { scrubSecrets } from './scrub-secrets.js'; +// Observe the scrubber's bounded fixed point, retaining its actual implementation, so one case can +// make it fail and see which value the scrubber fails closed with. +vi.mock('./fixed-point.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, toFixedPoint: vi.fn(actual.toFixedPoint) }; +}); + // Synthetic credentials assembled from fragments, never real provider material. const key = ['sk-', 'ant-', 'api03-', 'AbCdEf0123456789xyz'].join(''); const googleKey = ['AI', 'za', 'SyA1234567890abcdefghijklmnopqrstuv'].join(''); @@ -1304,3 +1312,36 @@ describe('PR95 review round 5: label pieces, a split ftp credential and the repe } }); }); + +describe('PR95 review round 6: the label bound behind a terminal sequence, fragments and failing closed', () => { + it('fails closed on nine labels behind a terminal sequence', () => { + const bound = (value: unknown) => expect(value, 'SECRETS_B4_LABEL_RUN_BOUND'); + // Normalization reads `ESC [` plus the run's first letter as one terminal sequence. + for (const label of ['Bearer', 'Basic']) { + for (const lead of ['Authorization: Bearer ', 'x-api-key: ', '']) { + bound(scrubSecrets(`${lead}\u001b[${label.repeat(9)} opaqueTOKEN123`)).toBe('[REDACTED]'); + } + } + }); + + it('masks a literal secret query name in a fragment', () => { + const fragment = (value: unknown) => expect(value, 'SECRETS_B6_QUERY_FRAGMENT'); + fragment(scrubSecrets('https://[::1]#id_token=Zk81Pq0aLm3nB7xY@127.0.0.1/cb')).toBe( + 'https://[::1]#id_token=[REDACTED]', + ); + fragment( + scrubSecrets( + 'redirect https://app.example/cb#access_token=Zk81Pq0aLm3nB7xYt2Rw&token_type=bearer', + ), + ).toBe('redirect https://app.example/cb#access_token=[REDACTED]&token_type=bearer'); + // A name that only begins like one, and an empty value, stay. + for (const kept of ['see https://docs.example/page#tokenizer=fast', 'page#token= next']) { + fragment(scrubSecrets(kept)).toBe(kept); + } + }); + + it('fails closed with its own mask when the scrub does not settle', () => { + vi.mocked(toFixedPoint).mockImplementationOnce((_pass, _text, failed) => failed); + expect(scrubSecrets('public text'), 'SECRETS_B6_FAIL_CLOSED').toBe('[REDACTED]'); + }); +}); diff --git a/packages/llm/src/scrub-secrets.ts b/packages/llm/src/scrub-secrets.ts index 1a45932f..5947c0e3 100644 --- a/packages/llm/src/scrub-secrets.ts +++ b/packages/llm/src/scrub-secrets.ts @@ -156,11 +156,12 @@ const NON_QUERY_SECRET_RULES: readonly SecretRule[] = [ /** * Exact literal query names with exact boundaries, so benign suffixes stay; no URL-name decoding. - * Their values join the merged masking pass, but no probe joins query names or ordinary query prose. + * A fragment carries them too (an OAuth implicit-flow `#access_token=`). Their values join the merged + * masking pass, but no probe joins query names or ordinary query prose. */ const QUERY_SECRET_RULE: SecretRule = { pattern: - /[?&](?:(?:x[-_])?api[-_]?key|api[-_]?secret|(?:access|secret)[-_]?key|(?:access|refresh|id|auth|session)[-_]?token|client[-_]?secret|key|token|auth|password|secret)=(?[^&\s#]+)/dgi, + /[?&#](?:(?:x[-_])?api[-_]?key|api[-_]?secret|(?:access|secret)[-_]?key|(?:access|refresh|id|auth|session)[-_]?token|client[-_]?secret|key|token|auth|password|secret)=(?[^&\s#]+)/dgi, mask: tokenGroup, }; @@ -169,7 +170,9 @@ const SECRET_RULES: readonly SecretRule[] = [...NON_QUERY_SECRET_RULES, QUERY_SE /** * Nine concatenated labels (`BearerBearer…`, possibly with quotes or escapes between them) exceed - * every bounded label run above, so the token after them cannot be told from them: fail closed. + * every bounded label run above, so the token after them cannot be told from them: fail closed. The + * raw text is checked as well as its normalized view, which can drop a label letter as the final + * byte of a terminal sequence (`ESC [` then `BearerBearer…`). */ const LABEL_RUN_PAST_BOUND = /(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)(?:[\\"']*(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)){8}/i; @@ -404,7 +407,7 @@ function hasSplitCredential(text: string, joined: string, rule: SecretRule): boo */ function scrubOnce(text: string): string { const normalized = normalizeSecretProbe(text); - if (LABEL_RUN_PAST_BOUND.test(normalized)) return MASKED; + if (LABEL_RUN_PAST_BOUND.test(normalized) || LABEL_RUN_PAST_BOUND.test(text)) return MASKED; const joined = /[\n\t]/.test(normalized) ? normalized.replace(/[\n\t]+/g, '') : undefined; for (const rule of NON_QUERY_SECRET_RULES) { const rawMasked = applyRule(text, rule); diff --git a/tools/causal-controls/provenance.test.mjs b/tools/causal-controls/provenance.test.mjs index 59ed7d0d..fd47946b 100644 --- a/tools/causal-controls/provenance.test.mjs +++ b/tools/causal-controls/provenance.test.mjs @@ -270,7 +270,7 @@ test('secrets workload corrections retain independent complete-count and reader- assert.equal(secretsControls.length, 110); assert.equal( secretsControls.reduce((count, control) => count + 1 + (control.witnesses ?? []).length, 0), - 483, + 491, ); for (const [id, file, expectedTests] of [ ['run-default-secret-reader', 'apps/cli/src/commands/run.test.ts', 84], @@ -313,13 +313,17 @@ test('secrets workload corrections retain independent complete-count and reader- 'core-userinfo-contact-host', 'core-userinfo-host-length', 'core-userinfo-host-anchors', - 'core-userinfo-query-extent', - 'core-userinfo-joined-extent', - 'core-split-userinfo-before-scheme', 'core-userinfo-split-scheme', 'core-userinfo-split-edges', 'core-fixed-point', - ].map((id) => [id, ['packages/core/src/tools/bounding.test.ts'], 167]), + // Round 6: core's merged ranges and the mechanisms they read through. + 'core-merged-ranges', + 'core-userinfo-query-zone', + 'core-key-value-nested', + 'core-key-value-through-masks', + 'core-scheme-nested-label', + 'core-mask-boundaries', + ].map((id) => [id, ['packages/core/src/tools/bounding.test.ts'], 185]), ...[ 'diagnostic-joined-before-masking', 'diagnostic-joined-original-boundary', @@ -346,7 +350,6 @@ test('secrets workload corrections retain independent complete-count and reader- 'diagnostic-header-lookbehind', 'diagnostic-header-escaped-separator', 'diagnostic-header-quote-run', - 'diagnostic-header-backslash-first', 'diagnostic-header-escaped-stop', 'diagnostic-label-token-not-label', 'diagnostic-label-quoted-label', @@ -361,15 +364,13 @@ test('secrets workload corrections retain independent complete-count and reader- (mechanism) => `diagnostic-label-${mechanism}`, ), 'diagnostic-header-concat-run', - 'diagnostic-header-stop-brace', - 'diagnostic-header-stop-quote', 'diagnostic-query-merge', 'diagnostic-mask-merge-extent', 'diagnostic-fixed-point', 'diagnostic-userinfo-authority-rest', 'diagnostic-userinfo-rest-length', 'diagnostic-url-split-retry', - ].map((id) => [id, ['packages/llm/src/scrub-secrets.test.ts'], 215]), + ].map((id) => [id, ['packages/llm/src/scrub-secrets.test.ts'], 218]), ...['diagnostic-inline-projection-order', 'diagnostic-original-control-boundary'].map((id) => [ id, ['apps/cli/src/render/sanitize.test.ts', 'apps/cli/src/process/render-error.test.ts'], diff --git a/tools/secrets-controls/README.md b/tools/secrets-controls/README.md index 28c61d88..9970385a 100644 --- a/tools/secrets-controls/README.md +++ b/tools/secrets-controls/README.md @@ -4,7 +4,7 @@ Step 4 first-run input matrix, Step 5 snapshot/export, Step 7 provider-header and PR #95 B2 redaction controls on physical source copies. `--mechanism ` supports diagnosis only; selected runs retain `complete: false`. The current manifest -declares **110 independent controls and 483 required named assertion witnesses**. +declares **110 independent controls and 491 required named assertion witnesses**. These counts describe the workload; a declaration is not evidence that its removal passed. Actual fixed/removal/restored results belong to the coordinator's summary. @@ -178,8 +178,8 @@ modern `sk-` body characters, joined-span application, the original-text left-bo check, split scheme labels, split API-header labels and raw `?`/`#` password recognition independently. Four LLM controls remove the before-masking LF/TAB probe, its original left-boundary check, split API-header recognition and raw-password recognition. -Every core phase executes the complete 167-case bounding file; every new LLM phase -executes the complete 215-case scrubber file. Independent ordinary tooling tests pin +Every core phase executes the complete 185-case bounding file; every new LLM phase +executes the complete 218-case scrubber file. Independent ordinary tooling tests pin these inventories, the 278-case overlap cohort, 180-case CLI diagnostic cohort and 18-case command-export cohort. Actual collection still has to match in every phase. @@ -260,7 +260,7 @@ previous-word prefilter and the rule's own boundary predicate; retaining either would leave the tested policy active. Header/key recovery removals still change their original-rule boundary only, and all removals retain the same canonical matchers and bounded raw/joined standalone visitors. Current collection is 25 -complete files / 1,473 cases, with each control's exact workload and named target +complete files / 1,494 cases, with each control's exact workload and named target checked separately. These are updated declarations and author checks, not new physical causal captures or a clean review round. R3 is preserved as NONCLEAN; later corrective acceptance belongs to the owning phase register. Prior selected @@ -355,6 +355,34 @@ the label rule's trailing `;`/`,`, its embedded stop's closing quote run and bou bound's case and LF/TAB letters, the exactly masked password, the escaped joined header name and both fixed-point helpers were each checked against the mutation they kill. +The sixth round found that round 5's value mirror rescanned the text from every query assignment +(quadratic at admitted size), still masked through a later credential's key name, label or PEM +marker, and was dropped by the split view, which treated a URL after any line break as split. +Core now works like the scrubber: every rule reports its ranges on one text and overlapping ranges +merge before anything is replaced, so no mask removes another rule's context. A key or parameter +in a URL userinfo's own text — all of an unambiguous userinfo, and a `user:password` before a raw +`?`/`#` — belongs to it; the key/value and opaque-query rules read the rest of an ambiguous +userinfo. The key/value rule resumes its scan at each value, so a key inside a value is read too, +and reads values through earlier masks, line breaks included; the scheme-label rule resumes after +each label; an LF/TAB-split header label, and a scheme label even where a key's value runs on over +it, may start right after a credential (one helper, `afterMasks`, fills the credentials before them +as applied masks would); and PEM blocks are found with one shared END search instead of a bounded +rescan after every BEGIN marker. Six controls pin the new mechanisms (`core-merged-ranges`, +`core-userinfo-query-zone`, `core-key-value-nested`, `core-key-value-through-masks`, +`core-scheme-nested-label`, `core-mask-boundaries`). The three round-5 core controls are gone with their mechanisms; +`core-joined-credential-spans`, `core-joined-original-boundary`, `core-userinfo-split-scheme`, +`core-userinfo-split-edges`, `diagnostic-label-run-bound` and `query-names` are re-anchored; and +three single-character pieces of the header rule (`diagnostic-header-stop-brace`, +`diagnostic-header-stop-quote`, `diagnostic-header-backslash-first`) are retired to make room, each +after its test was checked to still kill its mutation. Pinned by tests without a control: the +userinfo zone's colon and unambiguous branches, the opaque rule's parameter view, the fill's +boundary brackets, the PEM body bound (20,000 and 20,001), the scrubber's raw-text nine-label check +and `#` query names, and both maskers' fail-closed values, observed through a module mock of the +fixed-point helper that keeps its implementation. The split view still maps a URL's scheme past a +separator right before it; that only spares reporting the main view's range a second time, so the +output cannot show it and it has neither a test nor a control. The shared END search is pinned by +the admitted-size case deadline alone, like round 4's deadline-only guards. + The thin entry point uses the same coordinator, outcome and bounded process join as the filesystem family in [causal-controls](../causal-controls/README.md). The provider-header diagnostic control pins the store's fixed throwable message diff --git a/tools/secrets-controls/mutations.mjs b/tools/secrets-controls/mutations.mjs index ffb0bcb4..e2c4aa49 100644 --- a/tools/secrets-controls/mutations.mjs +++ b/tools/secrets-controls/mutations.mjs @@ -142,7 +142,7 @@ const mergedMasking = [ ' SECRET_RULES.flatMap((rule) => ruleRanges(text, rule)),', ' );', ].join('\n'); -const queryRule = (names) => String.raw`/[?&](?:${names})=(?[^&\s#]+)/dgi`; +const queryRule = (names) => String.raw`/[?&#](?:${names})=(?[^&\s#]+)/dgi`; const queryNames = String.raw`(?:x[-_])?api[-_]?key|api[-_]?secret|(?:access|secret)[-_]?key|(?:access|refresh|id|auth|session)[-_]?token|client[-_]?secret|key|token|auth|password|secret`; const overlapMerge = ' if (open !== undefined && range.start < open.end) {'; const mergeExtent = ' open.end = Math.max(open.end, range.end);'; @@ -255,18 +255,6 @@ const coreOriginalBoundary = String.raw` const candidates = /(^|[^\w])([A-Za-z] } } -function redactStandaloneCredentials(text: string): string { - const parts: string[] = []; - let copied = 0; - visitJoinedStandaloneCredentials(text, text, (start, end, replacement) => { - parts.push(text.slice(copied, start), replacement); - copied = end; - }); - if (parts.length === 0) return text; - parts.push(text.slice(copied)); - return parts.join(''); -} - /** * The finite header-label literal is the sole joined-header authority. Check ORIGINAL boundaries * before matching, so an embedded x-api-key/x-goog-api-key cannot consume an inner api-key start. @@ -301,25 +289,25 @@ function visitJoinedHeaderLabels(original: string, joined: string, mask: SecretS } /** - * LF/TAB may split finite credential shapes. Map recognized spans back onto the original text; - * ordinary spaces remain boundaries and public text is never globally normalized for identity. - * Token-character continuations after LF/TAB are intentionally part of a recognized credential. - * The two cursors advance monotonically, with no per-character offset array or recursive matcher. + * LF/TAB may split finite credential shapes. Map the spans recognized in the joined view back onto + * the original text and report the ones a removed separator actually splits; ordinary spaces remain + * boundaries and public text is never globally normalized for identity. Token-character + * continuations after LF/TAB are intentionally part of a recognized credential. The two cursors + * advance monotonically, with no per-character offset array or recursive matcher. */ -function redactJoinedCredentialSpans( +function visitSplitSpans( text: string, visit: (joined: string, mask: SecretSpanVisitor) => void, + report: SecretSpanVisitor, boundary?: RegExp, // URL userinfo owns a separator at either edge: one right after the scheme or before the @ splits it. ownsEdges = false, -): string { - if (!/[\n\t]/.test(text)) return text; +): void { + if (!/[\n\t]/.test(text)) return; const joined = text.replace(/[\n\t]+/g, ''); - const parts: string[] = []; const separators = /[\n\t]+/g; let next = separators.exec(text); let removed = 0; - let copied = 0; function advance(target: number, includeAtBoundary = false): number { while ( next !== null && @@ -330,9 +318,11 @@ function redactJoinedCredentialSpans( } return target + removed; } - visit(joined, (start, end, replacement, splitFrom) => { - const originalSplitFrom = splitFrom === undefined ? undefined : advance(splitFrom); + visit(joined, (start, end, replacement, url) => { + // A separator right before a URL's scheme is outside the URL, so it splits nothing. + const schemeStart = url === undefined ? undefined : advance(url.schemeStart, true); const originalStart = advance(start, !ownsEdges); + const ownsUntil = url === undefined ? undefined : advance(url.ownsUntil, ownsEdges); const originalEnd = advance(end, ownsEdges); if (boundary?.test(text[originalStart - 1] ?? '')) return;`; const withoutCoreOriginalBoundary = coreOriginalBoundary @@ -342,7 +332,7 @@ const withoutCoreOriginalBoundary = coreOriginalBoundary ) .replace(" if (boundary?.test(text[originalStart - 1] ?? '')) return;", ''); /** The complete core projection test file every core projection control runs. */ -const CORE_PROJECTION_CASES = 167; +const CORE_PROJECTION_CASES = 185; function coreProjectionControl(id, before, after, targets, marker = coreProjectionMarker) { return control( id, @@ -357,7 +347,7 @@ function coreProjectionControl(id, before, after, targets, marker = coreProjecti ); } /** The complete scrubber test file every LLM diagnostic control runs. */ -const LLM_SCRUB_CASES = 215; +const LLM_SCRUB_CASES = 218; function diagnosticControl(id, before, after, targets, marker = diagnosticMarker) { return control( id, @@ -1049,8 +1039,8 @@ export const mutations = [ ), coreProjectionControl( 'core-joined-credential-spans', - ' visit(joined, (start, end, replacement, splitFrom) => {', - " visit('', (start, end, replacement, splitFrom) => {", + ' visit(joined, (start, end, replacement, url) => {', + " visit('', (start, end, replacement, url) => {", [ 'masks every LF/TAB split without losing the surrounding public identity', ...splitCoreHeaders, @@ -1190,7 +1180,7 @@ export const mutations = [ // Nine concatenated labels exceed every bounded run, so the diagnostic fails closed. diagnosticControl( 'diagnostic-label-run-bound', - ' if (LABEL_RUN_PAST_BOUND.test(normalized)) return MASKED;', + ' if (LABEL_RUN_PAST_BOUND.test(normalized) || LABEL_RUN_PAST_BOUND.test(text)) return MASKED;', '', [ 'fails closed on nine concatenated labels and keeps eight in place', @@ -1229,10 +1219,6 @@ export const mutations = [ { separator: header.separator.replace(String.raw`\\*["']?\s*:`, String.raw`["']?\s*:`) }, ], ['diagnostic-header-quote-run', { prefix: String.raw`(?:\\*["'])?` }], - [ - 'diagnostic-header-backslash-first', - { token: header.token.replace(String.raw`[^\s"',;}\\]`, String.raw`[^\s"',;}]`) }, - ], ].map(([id, part]) => diagnosticControl( id, @@ -1242,18 +1228,6 @@ export const mutations = [ 'SECRETS_B3_HEADER_ESCAPES', ), ), - ...[ - ['diagnostic-header-stop-brace', String.raw`[\s"',;]`], - ['diagnostic-header-stop-quote', String.raw`[\s",;}]`], - ].map(([id, stop]) => - diagnosticControl( - id, - headerLine(), - headerLine({ stop: header.stop.replace(String.raw`[\s"',;}]`, stop) }), - ['ends a header token at a closing quote or brace'], - 'SECRETS_B4_HEADER_STOP', - ), - ), diagnosticControl( 'diagnostic-header-escaped-stop', headerLine(), @@ -1452,32 +1426,72 @@ export const mutations = [ ['bounds the host test after an ambiguous authority at admitted size'], 'SECRETS_B4_REST_LENGTH', ), - // Round 5: an ambiguous userinfo is masked with exactly the query values core's later rules read. + // Round 6: core's rules report ranges on one text and merge them, as the scrubber's do. An + // ambiguous userinfo's query is read by the later rules, a key or scheme label inside a value + // starts its own credential, a value runs through earlier masks, and a split header label, a + // scheme label or a credential may start right after another credential. coreProjectionControl( - 'core-userinfo-query-extent', - ' maskEnd = Math.max(maskEnd, laterQueryValueEnd(text, start + query, start + at));', - '', - ['masks an ambiguous userinfo with the query values its later rules would mask whole'], + 'core-merged-ranges', + ' if (run !== undefined && range.start < run.end) {', + ' if (run !== undefined && false) {', + [ + 'keeps every credential and its context when core masks overlap', + ...['\n', '\t', 'word\t', 'DATABASE_URL:\n'].map( + (prefix) => + `masks an ambiguous userinfo with its query values behind ${JSON.stringify(prefix)}`, + ), + 'masks an ambiguous userinfo whose scheme is split by an LF', + ], + 'SECRETS_B6_CORE_MERGE', + ), + coreProjectionControl( + 'core-userinfo-query-zone', + ' const ownsUntil = query < 0 ? at : colon >= 0 && colon < query ? query : 0;', + ' const ownsUntil = at;', + [ + 'masks an ambiguous userinfo with the query values its later rules would mask whole', + 'masks an admitted-size query value inside an ambiguous userinfo', + ], 'SECRETS_B5_CORE_QUERY_EXTENT', ), coreProjectionControl( - 'core-userinfo-joined-extent', - ' if (extend && query >= 0) {', - ' if (query >= 0) {', - ['keeps the context of a later credential after an ambiguous authority'], - 'SECRETS_B5_CORE_LATER_CONTEXT', + 'core-key-value-nested', + " if (delimiterAt - valueStart >= 6) mask(match.index, delimiterAt, '[redacted]');", + " if (delimiterAt - valueStart >= 6) {\n mask(match.index, delimiterAt, '[redacted]');\n key.lastIndex = delimiterAt;\n }", + ["reads a key inside another key's value"], + 'SECRETS_B6_CORE_NESTED_KEY', + ), + coreProjectionControl( + 'core-key-value-through-masks', + ' visitSecretKeyValues(names, fillRanges(text, masked), record);', + ' visitSecretKeyValues(names, names, record);', + ['reads a value through the masks before it, line breaks included'], + 'SECRETS_B6_CORE_VALUE_THROUGH_MASK', + ), + coreProjectionControl( + 'core-scheme-nested-label', + ' scheme.lastIndex = labelEnd;', + '', + [ + "reads a scheme label inside another label's token", + 'keeps nested values, labels and open PEM blocks linear at admitted size', + ], + 'SECRETS_B6_CORE_NESTED_LABEL', ), coreProjectionControl( - 'core-split-userinfo-before-scheme', - " const schemes = redactUrlUserinfo(keys).replace(SCHEME_CREDENTIAL, '$1 [redacted]');", - " const schemes = keys.replace(SCHEME_CREDENTIAL, '$1 [redacted]');", - ['masks the userinfo of a URL behind a scheme label, split or not, and a token glued to a URL'], - 'SECRETS_B5_CORE_SCHEME_ORDER', + 'core-mask-boundaries', + ' return fillRanges(text, ranges);', + ' return text;', + [ + 'joins a split header label right after a credential or a PEM block', + 'reads a scheme label or a credential glued to the end of another credential', + ], + 'SECRETS_B6_CORE_MASK_BOUNDARY', ), coreProjectionControl( 'core-userinfo-split-scheme', - " mask(start, maskEnd, '[redacted]', match.index);", - " mask(start, maskEnd, '[redacted]');", + ' schemeStart: match.index,', + ' schemeStart: start,', ['\n', '\t'].map( (separator) => `masks URL userinfo whose scheme is split by an LF or TAB (${JSON.stringify(separator)})`, @@ -1486,8 +1500,8 @@ export const mutations = [ ), coreProjectionControl( 'core-userinfo-split-edges', - ' const urls = redactJoinedCredentialSpans(text, visitUrlUserinfo, undefined, true);', - ' const urls = redactJoinedCredentialSpans(text, visitUrlUserinfo);', + ' visitSplitSpans(text, visitUrlUserinfo, report, undefined, true);', + ' visitSplitSpans(text, visitUrlUserinfo, report);', ['\n', '\t'].map( (separator) => `masks URL userinfo split at either of its edges (${JSON.stringify(separator)})`, From 082962e2ab47bb3225cd3c2d232091107d36252f Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Tue, 20 Oct 2026 22:34:28 +0300 Subject: [PATCH 16/97] fix(core): address the seventh PR95 review round MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 7 of the PR95 increment's review (at b89ac62e) confirmed 17 findings, two high. A URL userinfo's owned zone still hid real keys, the quoted-value pattern exhausted V8's RegExp stack at an admitted 8 MiB value, a projection that did not settle could give two calls one journal digest, and the secrets family's evidence export no longer fit its byte bound. All material findings are fixed. Core projection (packages/core/src/tools/bounding.ts): - Only a userinfo's username owns its keys: the text before its first ':' when no raw ?/# comes before that colon, nothing for a colonless userinfo, and in the split view never past the first LF/TAB. A key in the password or parameter position, or on the line after a split URL, now masks its value through the last '@' (jdbc:sqlserver://…;password=P@ss…, DATABASE_URL=https://h\n DB_PASSWORD=P@ss…), while a git remote such as https://x-access-token:@github.com/o/r keeps its host (PR95R7-SCRUB-1, SEC-R7-1). - Quoted values are found by a linear character scan instead of a repeated-group RegExp, so an exactly admitted 8 MiB value projects in about 50 ms instead of throwing RangeError after a write landed (PR95R7-SCRUB-2, SEC-R7-2). - A credential glued to the end of a fixed-length one (AKIA…ASIA…) is masked in the same pass, and a split span glued to the previous one is exempt from the original-boundary check, so such runs settle (PR95R7-GEN-1). - projectSecretShapedValue reports whether the journal projection settled (fixedPointOf), and makeSummary recognizes split shapes with the full fixed point (SEC-R7-3). - joinSplitHeaderLabels returns early when the text holds no ':' (PR95R7-CRIT-2; output-neutral). Registry (packages/core/src/tools/registry.ts): - A tiered effect whose journal projection does not settle is refused with a loop-correctable ToolArgsInvalidError before its prepare, its argument names masked, so the fail-closed constant never becomes a journal identity (PR95R7-GEN-1). Causal controls and CI: - The coordinator gains --shard index/count (round-robin, recorded in the summary, never complete), and the secrets job runs as a two-leg matrix whose legs export separately: one export of the 110-control family measured about 77 MiB against the unchanged 64-MiB bound (PR95R7-CRIT-1). Both shards ran locally with the export enabled: 1,087 files and 45.50 MiB, and 1,087 files and 45.48 MiB. - Ten controls: core-userinfo-username-zone, core-split-zone-cap, core-quoted-value-scan, core-standalone-chain, core-glued-span-boundary, core-summary-fixed-point, core-journal-settled, core-unsettled-refusal, and the two halves of the nine-label bound (diagnostic-label-run-normalized/-raw). The family is 120 controls / 501 targets and witnesses. Tests: discriminating cases for every witness above, the normalized nine-label view, values read through every kind of earlier mask, the merged run's replacement, the key rule's names and delimiters, and the refusal's masked names, each checked against the mutation it kills (TST-R7-1..5); the fixed-point mocks clear their history after each case, which drops the core file's heap from 725 MB to 130 MB (CONS-R7-1). Docs: built-in-tools.md, effect-journal.md, tool-registry.md and error-handling.md (DOC-R7-1 and the backstop's missing generic key/value rule), both controls READMEs, the phase register's round table (DOC-R7-2) and gates, and the status lines and freshness dates (DOC-R7-3). Refs: ADR-0113 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 21 +- AGENTS.md | 2 +- CLAUDE.md | 2 +- docs/reference/shared-core/built-in-tools.md | 33 ++- docs/reference/shared-core/effect-journal.md | 24 +- docs/reference/shared-core/tool-registry.md | 9 +- docs/roadmap/current.md | 4 +- docs/roadmap/deferred-tasks.md | 2 +- .../phase-2.5.5-hardening-and-remediation.md | 21 +- docs/standards/error-handling.md | 15 +- packages/core/src/tools/bounding.test.ts | 209 +++++++++++++++++- packages/core/src/tools/bounding.ts | 140 +++++++++--- packages/core/src/tools/fixed-point.test.ts | 16 +- packages/core/src/tools/fixed-point.ts | 18 +- packages/core/src/tools/registry.test.ts | 108 +++++++++ packages/core/src/tools/registry.ts | 55 ++++- packages/llm/src/scrub-secrets.test.ts | 19 +- tools/causal-controls/README.md | 7 + tools/causal-controls/provenance.mjs | 21 +- tools/causal-controls/provenance.test.mjs | 34 ++- tools/causal-controls/run.mjs | 12 + tools/ci/require-w8-acceptance.test.mjs | 7 +- tools/secrets-controls/README.md | 50 ++++- tools/secrets-controls/mutations.mjs | 116 +++++++++- 24 files changed, 828 insertions(+), 117 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6b4131bd..8b8bc4d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -279,15 +279,20 @@ jobs: retention-days: 7 secrets-mutations: - name: Wave 2 secrets causal controls + name: Wave 2 secrets causal controls (shard ${{ matrix.shard }} of 2) # The historical 35-control, three-phase capture exceeded the former aggregate budget. - # Wave 3's PR95 increment grows the cohort to 110 controls (2026-10-20): each scrubber - # control runs its eight-MiB-heavy file in three phases (about 24 s locally), so the job - # allowance matches the filesystem job's 60 minutes. Per-phase, case/hook, output and - # retained-evidence bounds remain unchanged (110 × 18 + 7 = 1,987 of 2,000 files); - # this comment does not certify execution. + # Wave 3's PR95 increment grows the cohort to 120 controls (2026-10-20): each scrubber + # control runs its eight-MiB-heavy file in three phases (about 24 s locally). One export + # of the whole cohort no longer fits the unchanged 64-MiB retained-evidence bound, so two + # legs each run and export a disjoint round-robin half (`--shard k/2`) under the + # unchanged per-phase, case/hook, output, 2,000-file and 64-MiB bounds; the allowance + # matches the filesystem job's 60 minutes. This comment does not certify execution. timeout-minutes: 60 runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: [1, 2] steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: @@ -304,12 +309,12 @@ jobs: - name: Run secrets removal and restoration controls env: RELAVIUM_ACCEPTANCE_EVIDENCE: ${{ runner.temp }}/wave2-acceptance - run: pnpm check:secrets-mutations + run: pnpm check:secrets-mutations --shard ${{ matrix.shard }}/2 - name: Retain bounded secrets control metadata if: ${{ always() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: wave2-secrets-${{ github.run_id }}-${{ github.run_attempt }} + name: wave2-secrets-${{ matrix.shard }}-of-2-${{ github.run_id }}-${{ github.run_attempt }} path: ${{ runner.temp }}/wave2-acceptance/secrets if-no-files-found: warn retention-days: 7 diff --git a/AGENTS.md b/AGENTS.md index 8c7daf07..a8697520 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -126,7 +126,7 @@ gate (red since at least PR #90) was cleared on 2026-10-19. A pre-implementation and the maintainer's [2026-10-19 rulings](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) give every open PR #95 finding an owner and record 29 increments (no new ADR). The PR95 increment (the scrubber residuals, including both security residuals) is implemented on `development` and under -review: rounds 1–6 were NONCLEAN and are corrected ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)), and acceptance +review: rounds 1–7 were NONCLEAN and are corrected ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)), and acceptance needs two consecutive clean rounds. Then Step 9a.1 (release map/custody substrate, #46/ADR-0111). `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. diff --git a/CLAUDE.md b/CLAUDE.md index 73de07db..3885fc5d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -215,7 +215,7 @@ gate (red since at least PR #90) was cleared on 2026-10-19. A pre-implementation and the maintainer's [2026-10-19 rulings](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) give every open PR #95 finding an owner and record 29 increments (no new ADR). The PR95 increment (the scrubber residuals, including both security residuals) is implemented on `development` and under -review: rounds 1–6 were NONCLEAN and are corrected ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)), and acceptance +review: rounds 1–7 were NONCLEAN and are corrected ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)), and acceptance needs two consecutive clean rounds. Then Step 9a.1 (release map/custody substrate, #46/ADR-0111). `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. diff --git a/docs/reference/shared-core/built-in-tools.md b/docs/reference/shared-core/built-in-tools.md index 4af9d1eb..fa50fdc6 100644 --- a/docs/reference/shared-core/built-in-tools.md +++ b/docs/reference/shared-core/built-in-tools.md @@ -104,12 +104,17 @@ leaves the authority ambiguous: the userinfo may also be a query that another ru host-shaped name follows its last `@`. This projection, like the backstop, finds every rule's ranges on the same text and merges overlapping ranges before it replaces anything, so no rule's mask removes the key name, scheme label, PEM marker or URL scheme another rule reads. Its -key/value and opaque-query rules read an ambiguous userinfo's keys and parameters from the raw `?` -or `#` on, or all of them when no `user:password` comes before it, and mask each value to its own -end (`https://gateway?password=P@ss.word1` → `https://[redacted]`). A key or parameter in a -userinfo's own text — all of an unambiguous userinfo, and a `user:password` before a raw `?` or -`#` — names a username or password, so its value does not run on through the public host and path -(`https://x-access-token:@github.com/o/r` → `https://[redacted]@github.com/o/r`). The projection +key/value and opaque-query rules read every key and parameter in a userinfo but its username's and +mask each value to its own end, through the last `@` when the value holds one +(`https://gateway?password=P@ss.word1` → `https://[redacted]`; +`jdbc:sqlserver://db.example:1433;user=sa;password=P@ssw0rd2024;encrypt=true` → +`jdbc:sqlserver://[redacted];encrypt=true`). The username is the text before the userinfo's first +`:` when no raw `?` or `#` comes before that colon; a userinfo with no such colon has none, and a +username never reaches past an LF or TAB. A key in the username names the user, so its value does +not run on through the public host and path +(`https://x-access-token:@github.com/o/r` → `https://[redacted]@github.com/o/r`), and a key on +the line after a split URL masks its value to its own end (`see https://\nDB_PASSWORD=P@ssw0rd123 now` → +`see https://\n[redacted] now`). The projection otherwise differs from that backstop by design: its authority also ends at `"`, `'`, `,`, `}` and `]`, so userinfo that contains one of them — an already-masked `[REDACTED]` password among them — is left as it is; it recognizes any scheme of up to 32 characters rather than a finite list; and a @@ -119,7 +124,12 @@ under every scheme. LF/TAB inside a recognized token-shaped credential extends the masked original span. An ambiguous token-alphabet continuation after LF/TAB is conservatively masked with that credential. An LF or TAB right after a URL's `://`, right before its userinfo's `@` or inside -its `scheme://` belongs to the split userinfo, which is masked as one span. A key's value runs +its `scheme://` belongs to the split userinfo, which is masked as one span. A credential glued to +the end of a fixed-length one (`AKIA…ASIA…`, `AKIA…ghp_…`) is a credential of its own, so a run of +them is masked whole in one pass. A variable-length token is masked to the end of its alphabet, so +a second token glued to it with no separator, whose prefix the first body takes only in part +(`ghp_ghp_`), is masked only through that part, and the rest of the second token stays +visible. A key's value runs through every span an earlier rule masks, line breaks included, as through an applied mask, so a quoted value that carries a split credential or a PEM block is masked whole. A key inside another key's value, and a scheme label inside another label's token, start a credential of their own @@ -129,10 +139,15 @@ block, as after an applied mask. A scheme label that takes a URL scheme as its token no longer hides that URL's password (`token postgres://app:pw@db` → `token [redacted]://[redacted]@db`). The projection repeats until its output stops changing, at most three passes, and becomes `[redacted]` if it has not settled, so -projecting a projection changes nothing; every rule's scan stays linear in the admitted text. +projecting a projection changes nothing. A tiered call whose journal projection has not settled is +refused before it is prepared ([tool registry](tool-registry.md#resolution--the-dispatch-lifecycle)), so that constant +never stands for two different arguments in the journal. Every rule's scan stays linear in the +admitted text, and none uses a backtracking pattern whose stack an admitted value can exhaust: a +quoted value is read by a character scan to its closing quote. Ordinary spaces and punctuation separate public prose; the masker does not discard the whole diagnostic because a credential appears. Supported split -shapes are recognized **before** complete-text one-line normalization, followed by +shapes are recognized **before** complete-text one-line normalization, by the projection's full +fixed point, followed by ordinary recognition and length clipping. This preserves multiline quoted-value recognition without exposing a split token's suffix. Public argument bytes are not globally whitespace-normalized for journal identity. The diff --git a/docs/reference/shared-core/effect-journal.md b/docs/reference/shared-core/effect-journal.md index 0126ad0c..46b2ef7e 100644 --- a/docs/reference/shared-core/effect-journal.md +++ b/docs/reference/shared-core/effect-journal.md @@ -370,8 +370,9 @@ inline-media redaction remain deliberate projection equality; raw credential byt added to the digest to distinguish calls. Recognized URL userinfo is masked through the authority's last `@`, preserving the host, -path, query and fragment; when it holds a raw `?` or `#`, the query values in it are masked to -their own ends as well. Recognized opaque URL-query values are masked through their delimiter, including values +path, query and fragment; the values of the keys and query parameters in it, its username's +excepted, are masked to their own ends as well, through that `@` when a value holds one. +Recognized opaque URL-query values are masked through their delimiter, including values longer than 512 characters. Quoted secret-key values honor escaped quotes and backslashes through the actual closing quote. A recognized value must not leave a credential suffix in the projection. This remains a finite shape detector, not a claim to identify arbitrary @@ -380,17 +381,22 @@ unlabelled secrets. Existing old digests are not retroactively erased or rewritt A raw `?` or `#` before the authority's `@` is public only after a host-shaped prefix that an `=` assignment follows ([built-in tools](built-in-tools.md#finite-diagnostic-credential-masking)). Wave 3's PR95 increment tightened that predicate without a new digest domain. It also merges -the ranges of every rule before replacing anything, so an ambiguous userinfo is masked together -with its query values and no mask removes another credential's context; reads a key or scheme -label inside another value as a credential of its own, and a value through earlier masks; masks -userinfo split by an LF or TAB at either edge or inside its `scheme://`; reads a split header -label or a scheme label right after a credential or a PEM block; and repeats the projection until -it is stable. +the ranges of every rule before replacing anything, so a userinfo is masked together with the +values of every key and query parameter in it but its username, which never reaches past an LF or +TAB, and no mask removes another credential's context; reads a key or scheme label inside another +value as a credential of its own, and a value through earlier masks; masks userinfo split by an LF +or TAB at either edge or inside its `scheme://`; masks a credential glued to the end of a +fixed-length one as a credential of its own; reads a split header label or a scheme label right +after a credential or a PEM block; and repeats the projection until it is stable. These change a `v5:` projection only for the arguments they project differently, such as userinfo previously treated as public or a credential an earlier mask had hidden: a resumed run whose committed effect carried one now computes a different digest and refuses matching prepare instead of replaying, which is the fail-closed outcome of any digest difference. Every other -`v5:` row keeps its digest. +`v5:` row keeps its digest. A projection that has not settled after three passes becomes the +constant `[redacted]` in diagnostics, but it never becomes a journal identity: the registry +refuses such a tiered call with `ToolArgsInvalidError` before its prepare +([tool registry](tool-registry.md#resolution--the-dispatch-lifecycle)), so two different calls +cannot share it and replay one another's results. A recognized structural credential key masks its entire value before traversal. A partial inner text match does not make the remaining value public. Only an anchored Bearer, Basic diff --git a/docs/reference/shared-core/tool-registry.md b/docs/reference/shared-core/tool-registry.md index b00a6230..bf2febe5 100644 --- a/docs/reference/shared-core/tool-registry.md +++ b/docs/reference/shared-core/tool-registry.md @@ -256,7 +256,12 @@ host is touched once, in the middle. 8. **Mark the result untrusted** (§Untrusted-data taint) and hand the structured `tool_call` / `tool_result` data + its taint/secret markers to the bus's single translation point ([ADR-0036](../../decisions/0036-run-loop-substrate-event-bus-and-execution-host.md)) for `agent:tool_call` / `agent:tool_result` emission. The durable [effect bracket](effect-journal.md) prepares a tiered call after approval and settles it -after model-facing bounding. Immediately before step 5, after any asynchronous approval and preparation, +after model-facing bounding. Its prepare carries the journal projection of the effective arguments; +when that projection has not settled within three passes (credential-shaped text with no stable +redacted form, [built-in-tools.md](built-in-tools.md#finite-diagnostic-credential-masking)), the call +is refused with a loop-correctable `ToolArgsInvalidError` before anything is prepared or dispatched. It +names the affected arguments through the same masker, since an open schema lets the model choose an +argument's name. Immediately before step 5, after any asynchronous approval and preparation, the registry invokes the optional trusted `ctx.beforeDispatch(toolId)` synchronously for **every actual dispatch**, including unjournaled tools. No asynchronous work may occur in that hook. It uses the resolved registry id and does not run for a retained-result replay. A refusal proves no dispatch started: a prepared @@ -480,7 +485,7 @@ codes by [sse-event-schema.md](../contracts/sse-event-schema.md#error-code-taxon | `UnknownToolError` | id not an exact match | `tool_failed` | fatal (loop-correctable first) | | `ToolPolicyError` | a guardrail / grant denial — `not_granted`, `provider_executed`, `command_not_allowed`, `domain_not_allowed`, `insecure_url`, `gate_required`, `media_scope_denied` (the full `ToolPolicyDenyReason` union; `media_scope_denied` is `read_media`'s scope-set denial, [ADR-0044](../../decisions/0044-media-access-governance-read-media-save-to-cost.md) §1) | `tool_denied` | **fatal** — EXCEPT `media_scope_denied` is `recoverable` (a Step-14 SCOPE denial; see the note below) | | `ToolDeniedByUserError` | an interactive **per-tool approval** denial ([ADR-0057](../../decisions/0057-cli-chat-modes-and-per-tool-approval.md) EA3) — `user_rejected` (rejected by the user / mode policy), `no_approval_hook` (fail-closed: a governed dispatch under an active regime with no confirm hook wired), `approval_error` (fail-closed: the hook threw a non-abort error, so consent could not be obtained) | `tool_denied` | **fatal** (never retried; not loop-correctable — re-asking re-prompts/re-denies, like `ToolPolicyError`) | -| `ToolArgsInvalidError` | effective args fail `parseArgs` / secret-taint | `validation` | fatal (loop-correctable first) | +| `ToolArgsInvalidError` | effective args fail `parseArgs` / secret-taint, or a tiered call's journal projection does not settle | `validation` | fatal (loop-correctable first) | | `ToolUnavailableError` | the required `ToolHost` capability is absent (host/config gap, not the model's fault) | `tool_unavailable` | **fatal** (names the tool + the unwired arm actionably — never a bare `internal`; EA1, [ADR-0055](../../decisions/0055-cli-host-capability-seam-tool-environment-factory.md)) | | `ToolExecutionError` | the host capability threw a non-cancel error (cause kept off the message, for logs) | `tool_failed` | retryable (node budget) | | *(AbortSignal abort)* | the run was cancelled mid-tool | `cancelled` | fatal (cancel path, not `tool_failed`) | diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index 750d0813..4c3cd756 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -1,7 +1,7 @@ # Current state > Status: Living -> Last updated: 2026-10-19 +> Last updated: 2026-10-20 - **Related**: [README.md](README.md), [phases/phase-2.5-cli-consolidation.md](phases/phase-2.5-cli-consolidation.md), [phases/phase-2.5.5-hardening-and-remediation.md](phases/phase-2.5.5-hardening-and-remediation.md), [phases/phase-2-cli.md](phases/phase-2-cli.md), [deferred-tasks.md](deferred-tasks.md), [../project-structure.md](../project-structure.md), [../tech-stack.md](../tech-stack.md) @@ -165,7 +165,7 @@ Wave 3 implementation continues on `development`: Steps 1–8 are merged. A pre- of Steps 9–19 and the maintainer's rulings are recorded in the [2026-10-19 rulings](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) (no new ADR; 29 recorded increments). **The PR95 increment is implemented on `development` and under review** — the PR #95 -scrubber residuals, including the two security residuals; rounds 1–6 were NONCLEAN and are corrected, as its +scrubber residuals, including the two security residuals; rounds 1–7 were NONCLEAN and are corrected, as its [implementation checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20) records, and acceptance needs two consecutive clean rounds after the last material fix. **Step 9a.1** follows, the first increment of the release map/custody substrate (#46, ADR-0111), whose 9a.1–9d increments carry a dedicated archive/crypto security sitting. Steps 10–19 follow the diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index 4a60efce..bc2a7f07 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -2,7 +2,7 @@ > Status: Living -> Last updated: 2026-10-19 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Steps 9–19 not started). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings. Existing approved deferrals remain. +> Last updated: 2026-10-20 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Steps 9–19 not started; the PR95 increment is under review on `development`). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings. Existing approved deferrals remain. > The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) records completed prerequisites; full M/N lifecycle and other scheduled work remain with their owners. > > Earlier triage: the Phase-2.6 rewrite triaged every open item, the 2026-07-19 full-project diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 5c58675f..35f6d487 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -9424,8 +9424,8 @@ carries the same owners. the CLI's authored-content detector, the secrets causal controls and the canonical documents. Base `9902e0a0`; commits `38e10d42` (Docs-0, the rulings), `dda3b45e` (implementation) and the corrections `ef3ded0d`, `bdd292a6`, `6fb7e9d9`, `b2c50e88`, `6b2e94dd` (round 5, which added this -section) and the round-6 commit. The commits stay local on `development` until the increment is -accepted. +section), `b89ac62e` (round 6) and the round-7 commit. The commits stay local on `development` until +the increment is accepted. **Review rounds.** Each round is a NEW independent complete cumulative review of `9902e0a0..HEAD`, every reviewer in its own worktree detached at the reviewed head: scrubber correctness, tests and causal @@ -9435,20 +9435,25 @@ stay local and Git-ignored. | Round | Reviewed head | Verdict | Verified findings | Main corrections | | --- | --- | --- | --- | --- | -| 1 | `dda3b45e` | NONCLEAN | 15 (2 high) | the host-shaped prefix bounded at 260 characters before the RegExp, which had exhausted V8's stack at admitted size, and anchored; stacked labels; already-masked userinfo kept | +| 1 | `dda3b45e` | NONCLEAN | 15 (2 high) | the host-shaped prefix bounded at 260 characters before the RegExp, which had exhausted V8's stack at admitted size, with its anchors and port bound pinned by tests; stacked labels; already-masked userinfo kept | | 2 | `ef3ded0d` | NONCLEAN | 15 | quote-closed labels, bounded label stacking, masked tokens as fixed points | | 3 | `bdd292a6` | NONCLEAN | 17 | sequential rule application replaced by one merged, token-only masking pass | | 4 | `6fb7e9d9` | NONCLEAN (integration clean) | 19 | the literal query names in the merged pass; concatenated label runs and the nine-label bound; one standalone label rule for both labels; a bounded fixed point in both maskers; the secrets CI job allowance becomes 60 minutes | | 5 | `b2c50e88` | NONCLEAN | 20 (2 high) | core's round-4 authority extent erased the context its sequential rules read, so core now masks an ambiguous userinfo with exactly the query values its later rules mask whole; its split view masks URL userinfo before scheme labels; the LLM split scan retries a kept candidate glued across a line break; tested fixed-point helpers; untested pattern pieces gained discriminating tests | | 6 | `6b2e94dd` | NONCLEAN | 21 (9 high) | round 5's value mirror was quadratic and still erased context, and its split view dropped the value mask; core now merges every rule's ranges on one text, as the scrubber does, reads a key or scheme label inside another value and a value through earlier masks, reads a split header label or a scheme label right after a credential, and scans PEM blocks linearly; the scrubber's nine-label bound also reads the raw text and its literal query names follow `#`; both fail-closed values are tested | +| 7 | `b89ac62e` | NONCLEAN | 17 (2 high) | only a userinfo's username owns its keys, never past an LF or TAB, so a key in its password or parameter position or on the next line masks its value through the last `@`; quoted values are found by a character scan that an admitted 8-MiB value cannot exhaust; a credential glued to a fixed-length one is masked in the same pass; summaries use the projection's fixed point; a tiered call whose journal projection does not settle is refused before its prepare; the secrets controls run as two CI shards so that each evidence export fits the unchanged 64-MiB bound, which the single export had exceeded; untested pieces gained discriminating tests | **Author gates and controls.** Every corrected head passed `pnpm run ci` and the affected suites under `CI=true GITHUB_ACTIONS=true`, plus local differential, exhaustive and admitted-size harnesses that -found no new leak against the previous head. The secrets family is 110 controls / 330 phases / 491 -targets and witnesses, projecting 1,987 of the unchanged 2,000 retained files; -[tools/secrets-controls/README.md](../../../tools/secrets-controls/README.md) records each round's -control changes and, from round 5, the rule that pieces inside a mechanism's pattern are pinned by -discriminating tests rather than by new controls. +found no new leak against the previous head. Until round 7 no author gate ran the evidence export, +which only the dedicated CI job enables, so the 110-control family's single export, about 77 MiB +against the unchanged 64-MiB bound, would have failed that job. From round 7 the family is 120 +controls / 360 phases / 501 targets and witnesses, run as two 60-control shards that each project +1,087 of the unchanged 2,000 retained files; both shards ran locally with +`RELAVIUM_ACCEPTANCE_EVIDENCE` set, and each export held 1,087 files and measured 45.50 MiB and +45.48 MiB, its final record included. [tools/secrets-controls/README.md](../../../tools/secrets-controls/README.md) +records each round's control changes and, from round 5, the rule that pieces inside a mechanism's +pattern are pinned by discriminating tests rather than by new controls. **Remaining gates.** The increment is not independently accepted: two NEW consecutive clean complete cumulative rounds after the last material fix, with a clean separate security sitting, remain. Then diff --git a/docs/standards/error-handling.md b/docs/standards/error-handling.md index 9ff7f1e9..8ce0979a 100644 --- a/docs/standards/error-handling.md +++ b/docs/standards/error-handling.md @@ -169,8 +169,9 @@ content. any scheme; an empty password, or text glued after a masked one, is still masked. The authored-content detector's projection of `postgres://app:{{secrets.password}}@db` to `postgres://app:[REDACTED]@db` is therefore a fixed point. Core's journal projection applies the - same `?`/`#` predicate and also merges its rules' ranges, but masks an ambiguous userinfo together - with the query values in it, each to its own end, rather than through the authority's end + same `?`/`#` predicate and also merges its rules' ranges, but masks a userinfo together with the + values of the keys and query parameters in it, its username's excepted, each to its own end, + rather than through the authority's end ([built-in tools](../reference/shared-core/built-in-tools.md#finite-diagnostic-credential-masking)). - **Split and normalized forms.** Every rule except the literal query names also probes the terminal-normalized text; a credential revealed only by normalization masks the **whole** @@ -211,9 +212,15 @@ content. whose quotes are themselves escaped as `\u0022` are outside the finite shapes. In a JSON-encoded diagnostic, an escape written as two characters (`\n`, `\t`, `\r`) and glued before a key prefix, a header name or a label defeats that shape's word boundary, so the credential after it stays - visible (`"key rejected:\nsk-…"`); core's journal projection shares this limit. A header value + visible (`"key rejected:\nsk-…"`). Core's journal projection shares this limit for key prefixes + and scheme labels; its key/value rule's word-character prefix still reads a header or secret-key + name glued after such an escape (`"err:\napi-key: …"`) and masks the value with it. A header value that begins with `,`, `;` or `}` and has no label before it is an empty token, so the text after - it stays visible. + it stays visible. The backstop has no generic key/value rule: a secret key inside a URL authority + whose value holds `@` — after `;` (`https://svc;a=1;password=P@ss…`), after `&` unless it is one + of the literal secret query names, or on the next line of a JSON-encoded file whose break is the + escape `\n` — leaves its value's text after the last `@` visible as the host. Core's key/value + rule reads that key and masks its value. ## No silent catches diff --git a/packages/core/src/tools/bounding.test.ts b/packages/core/src/tools/bounding.test.ts index d74f42c7..bc1ad4fd 100644 --- a/packages/core/src/tools/bounding.test.ts +++ b/packages/core/src/tools/bounding.test.ts @@ -1,22 +1,33 @@ import { scrubSecrets } from '@relavium/llm/scrub-secrets'; import type { AbortSignalLike } from '@relavium/shared'; -import { describe, expect, it, vi } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { boundForModel, redactInlineMedia, redactSecretShapedText, + projectSecretShapedValue, redactSecretShapedValue, utf8ByteLength, } from './bounding.js'; -import { toFixedPoint } from './fixed-point.js'; +import { fixedPointOf, toFixedPoint } from './fixed-point.js'; import type { ToolHost, ToolResultLimits } from './types.js'; // Observe the projection's bounded fixed point, retaining its actual implementation, so one case can // make it fail and see which value the projection fails closed with. vi.mock('./fixed-point.js', async (importOriginal) => { const actual = await importOriginal(); - return { ...actual, toFixedPoint: vi.fn(actual.toFixedPoint) }; + return { + ...actual, + fixedPointOf: vi.fn(actual.fixedPointOf), + toFixedPoint: vi.fn(actual.toFixedPoint), + }; +}); + +// The observation must not keep every admitted-size input and projection alive for the whole file. +afterEach(() => { + vi.mocked(fixedPointOf).mockClear(); + vi.mocked(toFixedPoint).mockClear(); }); const BIG: ToolResultLimits = { maxBytes: 50_000, maxLines: 2000 }; @@ -1743,3 +1754,195 @@ describe('PR95 review round 6: one merged pass of every core rule', () => { expect(redactSecretShapedText('public text'), 'SECRETS_B6_CORE_FAIL_CLOSED').toBe('[redacted]'); }); }); + +describe('PR95 review round 7: userinfo ownership, admitted quoted values, glued credentials', () => { + const projects = (marker: string, pairs: readonly (readonly [string, string, string])[]) => { + for (const [input, expected, secret] of pairs) { + const output = redactSecretShapedText(input); + expect(output, marker).toBe(expected); + expect(output.includes(secret), marker).toBe(false); + expect(redactSecretShapedText(output), marker).toBe(output); + } + }; + + it("owns no text past a split userinfo's first LF or TAB", () => { + projects('SECRETS_B7_CORE_SPLIT_ZONE', [ + [ + 'see https://\nDATABASE_PASSWORD=P@ssw0rd123 now', + 'see https://[redacted] now', + 'ssw0rd123', + ], + ['x https://\tSERVICE_TOKEN_VALUE=Adm1n@2024x', 'x https://[redacted]', '2024x'], + [ + 'DATABASE_URL=https://db.internal.example.com\nDB_PASSWORD=P@ssw0rd123\n', + 'DATABASE_URL=https://[redacted]\n', + 'ssw0rd123', + ], + [ + 'REDIS_URL=redis://cache.internal:6379\nREDIS_PASSWORD=r3d1s@Passw0rd\n', + 'REDIS_URL=redis://[redacted]\n', + 'Passw0rd', + ], + // A username-shaped key on the next line is no username of this URL. + ['see https://\nDB_PASSWORD:P@ssw0rd123 now', 'see https://[redacted] now', 'ssw0rd123'], + ['x https://\tSERVICE_TOKEN:Adm1n@2024x', 'x https://[redacted]', '2024x'], + ]); + }); + + it("reads a key in a userinfo's password or parameter position", () => { + projects('SECRETS_B7_CORE_USERNAME_ZONE', [ + [ + 'jdbc:sqlserver://db.example:1433;databaseName=app;user=sa;password=P@ssw0rd2024;encrypt=true', + 'jdbc:sqlserver://[redacted];encrypt=true', + 'ssw0rd2024', + ], + ['https://svc;a=1;password=P@ssw0rd123', 'https://[redacted]', 'ssw0rd123'], + ['mysql://h.example;password=Secr3tValue@x', 'mysql://[redacted]', 'Secr3tValue'], + [ + '{"content":"REDIS_URL=redis://cache.internal:6379\\nREDIS_PASSWORD=r3d1s@Passw0rd\\n"}', + '{"content":"REDIS_URL=redis://[redacted]"}', + 'Passw0rd', + ], + // A colon after a raw ?/#, or before any username, marks no user. + ['https://password=abcdefgh?x:y@host.example/path', 'https://[redacted]', 'abcdefgh'], + ['https://:password=abcdefghij?x@host.example/p', 'https://[redacted]', 'abcdefghij'], + ]); + }); + + it('keeps the host when only the username names a secret, split after its colon or not', () => { + const host = (value: unknown) => expect(value, 'SECRETS_B7_CORE_USERNAME_ZONE'); + host( + redactSecretShapedText( + 'https://x-access-token:\nghp_abcdefghijklmnopqrstuvwxyz0123@github.com/o/r', + ), + ).toBe('https://[redacted]@github.com/o/r'); + // Split inside the username, the key after the break is read and masks the host: fail-closed. + host( + redactSecretShapedText( + 'https://x-access-\ntoken:ghp_abcdefghijklmnopqrstuvwxyz0123@github.com/o/r', + ), + ).toBe('https://[redacted]'); + }); + + it('projects exactly admitted-size quoted values without exhausting the RegExp stack', () => { + const size = 8 * 1024 * 1024; + for (const [input, expected] of [ + [`token="${'a'.repeat(size - 7)}`, undefined], + [`password="${'a'.repeat(size - 11)}"`, '[redacted]'], + [`password='${'a'.repeat(size - 11)}'`, '[redacted]'], + [`{"access_token":"${'a'.repeat(size - 19)}"}`, '{"[redacted]}'], + ] as const) { + let output = ''; + expect(() => { + output = redactSecretShapedText(input); + }, 'SECRETS_B7_CORE_QUOTED_SCAN').not.toThrow(); + // An unclosed quote is no value: the text stays as it is. + expect(output === (expected ?? input), 'SECRETS_B7_CORE_QUOTED_SCAN').toBe(true); + } + }, 15_000); + + it("keeps quoted values on one line and pins the key rule's names and delimiters", () => { + const keys = (value: unknown) => expect(value, 'SECRETS_B7_CORE_KEY_PIECES'); + for (const input of [ + 'passwd=hunter2hunter2', + 'access_key=hunter2hunter2', + 'access-key: hunter2hunter2', + 'private_key=hunter2hunter2', + 'private-key: hunter2hunter2', + 'api_key_id=abcdef123456', + ]) { + keys(redactSecretShapedText(input)).toBe('[redacted]'); + } + keys(redactSecretShapedText('password=hunter2hunter2&next=public')).toBe( + '[redacted]&next=public', + ); + keys(redactSecretShapedText('token=abcdefgh,next')).toBe('[redacted],next'); + // A quoted value never crosses a line break, and a key must start within its word's first 33 characters. + for (const kept of [ + 'password="abc\ndef" tail', + "password='abc\rdef' tail", + `x${'y'.repeat(40)}token=listed`, + ]) { + keys(redactSecretShapedText(kept)).toBe(kept); + } + }); + + it('masks a run of glued fixed-length ids in one pass, split or not', async () => { + const ids = 'AKIAAAAAAAAAAAAAAAAAASIABBBBBBBBBBBBBBBBAKIACCCCCCCCCCCCCCCCASIADDDDDDDDDDDDDDDD'; + projects('SECRETS_B7_CORE_CHAIN', [ + [`${ids} public tail`, `${'[redacted]'.repeat(4)} public tail`, 'DDDDDDDD'], + [ + 'AKIAABCDEFGHIJKLMNOPghp_AAAAAAAAAAAA\nBBBBBBBBBBBBBBBB done', + '[redacted][redacted] done', + 'BBBBBBBB', + ], + [ + 'AKIAABCDEFGHIJKLMNOPsk-proj-AAAAAAAAAAAAAAAA\nBBBBBBBBBBBB done', + '[redacted][redacted] done', + 'BBBBBBBB', + ], + ]); + const summary = await boundForModel( + 'AKIAABCDEFGHIJKLMNOPsk-proj-AAAAAAAAAAAAAAAA\tBBBBBBBBBBBB done', + BIG, + {}, + ); + expect(summary.summary, 'SECRETS_B7_CORE_CHAIN').toBe('[redacted][redacted] done'); + }); + + it("summarizes with the projection's fixed point before one-line normalization", async () => { + const result = await boundForModel( + 'g://s.hos#mHPGE\nAXXQ8@?q==a@b.exam/user:3eYPUzfxo', + BIG, + {}, + ); + expect(result.summary, 'SECRETS_B7_CORE_SUMMARY').toBe('g://[redacted]@?q=[redacted]'); + }); + + it('reads a value through earlier masks of every kind, in text order', () => { + projects('SECRETS_B6_CORE_VALUE_THROUGH_MASK', [ + ['token=xyz?sig=AAAAAAAAAAAAAAAAAAAAAAAA;BBB#TAILSECRET', '[redacted]', 'TAILSECRET'], + [ + 'password=https://svc?sig=AAAAAAAAAAAAAAAAAAAAAAAAAAA&x=1@hostsecretTail', + '[redacted]', + 'secretTail', + ], + [ + 'password=-----BEGIN RSA PRIVATE KEY-----\nabc\n-----END RSA PRIVATE KEY-----TAIL https://u:p@h', + '[redacted] https://[redacted]@h', + 'TAIL', + ], + ]); + }); + + it('keeps a scheme label and its space when its token is also a standalone credential', () => { + projects('SECRETS_B6_CORE_MERGE', [ + ['Bearer sk-abcdefghijklmnopqrstuvwx', 'Bearer [redacted]', 'abcdefghij'], + [ + 'Authorization: Bearer sk-proj-abcdefghijklmnopqrstuvwx', + '[redacted] [redacted]', + 'abcdefghij', + ], + ['token sk-abcdefghijklmnopqrstuvwx', 'token [redacted]', 'abcdefghij'], + ]); + }); + + it('reports whether a journal projection settled', () => { + const journal = (value: unknown) => expect(value, 'SECRETS_B7_CORE_JOURNAL_SETTLED'); + const content = + 'AKIAAAAAAAAAAAAAAAAAASIABBBBBBBBBBBBBBBBAKIACCCCCCCCCCCCCCCCASIADDDDDDDDDDDDDDDD\nexpect(scan(x)).toBe(4)\n'; + journal(projectSecretShapedValue({ path: 'test/fixtures/keys.txt', content })).toEqual({ + value: { + path: 'test/fixtures/keys.txt', + content: `${'[redacted]'.repeat(4)}\nexpect(scan(x)).toBe(4)\n`, + }, + settled: true, + }); + // A string whose projection does not settle projects to the mask and is reported. + vi.mocked(fixedPointOf).mockImplementationOnce(() => undefined); + journal(projectSecretShapedValue('drifting text')).toEqual({ + value: '[redacted]', + settled: false, + }); + }); +}); diff --git a/packages/core/src/tools/bounding.ts b/packages/core/src/tools/bounding.ts index 02973ab7..c22c65ee 100644 --- a/packages/core/src/tools/bounding.ts +++ b/packages/core/src/tools/bounding.ts @@ -23,7 +23,7 @@ import { utf8ByteLength } from '@relavium/shared'; // while the IMPLEMENTATION lives in `@relavium/shared` — see its header for why it moved. export { utf8ByteLength }; -import { toFixedPoint } from './fixed-point.js'; +import { fixedPointOf, toFixedPoint } from './fixed-point.js'; import type { ToolHost, ToolResultLimits } from './types.js'; export interface BoundedResult { @@ -171,12 +171,6 @@ const SCHEME_CREDENTIAL = // `visitSecretKeyValues`. const SECRET_KEY = /\b[\w-]{0,32}(?:password|passwd|secret|token|api[_-]?key|authorization|access[_-]?key|private[_-]?key|client[_-]?secret)[\w-]{0,16}["']?\s*[=:]\s*/gi; -// A QUOTED value ends at the first UNESCAPED matching quote, including JSON's escaped quotes/backslashes; -// otherwise a valid password such as `a\"secret-tail` left its tail exposed. The two alternatives inside -// each repetition are disjoint: a backslash consumes exactly the following non-newline character, while -// the ordinary class excludes it. No ambiguous nested repetition/backref is needed. -const DOUBLE_QUOTED_VALUE = /"(?:[^"\\\r\n]|\\[^\r\n])*"/y; -const SINGLE_QUOTED_VALUE = /'(?:[^'\\\r\n]|\\[^\r\n])*'/y; /** * One recognized span of the projection's input and the text that replaces it. Every rule reports @@ -194,7 +188,7 @@ interface SecretRange { * A URL userinfo's context. `schemeStart` is where its identifying prefix starts: a removed LF/TAB * anywhere from there on splits the credential, so a separator inside `scheme://` counts even though * only the userinfo is replaced. Up to `ownsUntil` the userinfo owns its text: a key or query - * parameter there names a username or password, not a credential of its own. + * parameter there names the user, not a credential of its own. */ interface UrlUserinfo { readonly schemeStart: number; @@ -286,6 +280,16 @@ function visitJoinedStandaloneCredentials( if (match === null) continue; matchedUntil = JOINED_STANDALONE_CREDENTIAL.lastIndex; mask(start, matchedUntil, '[redacted]'); + // A credential glued to the end of this one starts one of its own, as after an applied mask, so a + // run of fixed-length ids (AKIA…ASIA…) is masked whole in one pass. + for (let glued = matchedUntil; /[A-Za-z]/.test(joined[glued] ?? ''); glued = matchedUntil) { + JOINED_STANDALONE_CREDENTIAL.lastIndex = 0; + if (!JOINED_STANDALONE_CREDENTIAL.test(preview(glued))) break; + JOINED_STANDALONE_CREDENTIAL.lastIndex = glued; + if (JOINED_STANDALONE_CREDENTIAL.exec(joined) === null) break; + matchedUntil = JOINED_STANDALONE_CREDENTIAL.lastIndex; + mask(glued, matchedUntil, '[redacted]'); + } } } @@ -352,19 +356,30 @@ function visitSplitSpans( } return target + removed; } + let previousEnd = -1; visit(joined, (start, end, replacement, url) => { // A separator right before a URL's scheme is outside the URL, so it splits nothing. const schemeStart = url === undefined ? undefined : advance(url.schemeStart, true); const originalStart = advance(start, !ownsEdges); const ownsUntil = url === undefined ? undefined : advance(url.ownsUntil, ownsEdges); const originalEnd = advance(end, ownsEdges); - if (boundary?.test(text[originalStart - 1] ?? '')) return; + // A span that starts where the visitor's previous one ended is glued to a credential, which bounds + // it as an applied mask would. + const glued = originalStart === previousEnd; + previousEnd = originalEnd; + if (!glued && boundary?.test(text[originalStart - 1] ?? '')) return; if (!/[\n\t]/.test(text.slice(schemeStart ?? originalStart, originalEnd))) return; + // A split userinfo owns nothing past its first removed separator: the next line may hold a key + // of its own (`https://\nDB_PASSWORD=P@ss`). + const split = + ownsUntil === undefined ? -1 : text.slice(originalStart, ownsUntil).search(/[\n\t]/); report( originalStart, originalEnd, replacement, - schemeStart === undefined || ownsUntil === undefined ? undefined : { schemeStart, ownsUntil }, + schemeStart === undefined || ownsUntil === undefined + ? undefined + : { schemeStart, ownsUntil: split < 0 ? ownsUntil : originalStart + split }, ); }); } @@ -395,7 +410,8 @@ function afterMasks(text: string, ranges: readonly SecretRange[]): string { * starts at a word boundary, and a credential or a PEM block right before it is one (`afterMasks`). */ function joinSplitHeaderLabels(text: string): string { - if (!/[\n\t]/.test(text)) return text; + // A label ends in ':' (visitJoinedHeaderLabels) and no fill adds one, so a text without one has none. + if (!/[\n\t]/.test(text) || !text.includes(':')) return text; const masks: SecretRange[] = []; const mask: SecretSpanVisitor = (start, end, replacement) => { masks.push({ start, end, replacement }); @@ -614,10 +630,32 @@ function visitSchemeCredentials(text: string, mask: SecretSpanVisitor): void { } } +/** + * Where the quoted value that opens at `start` ends, or -1 when it does not close: just past the + * first UNESCAPED matching quote, including JSON's escaped quotes/backslashes (otherwise a valid + * password such as `a\"secret-tail` left its tail exposed), and never across a CR or LF. A backslash + * consumes exactly the next character. A plain scan, because a RegExp repeat over the value keeps + * backtracking state per character and exhausts V8's stack on an admitted-size value. + */ +function quotedValueEnd(text: string, start: number): number { + const quote = text[start]; + for (let index = start + 1; index < text.length; index += 1) { + const char = text[index]; + if (char === quote) return index + 1; + if (char === '\r' || char === '\n') return -1; + if (char === '\\') { + const next = text[index + 1]; + if (next === undefined || next === '\r' || next === '\n') return -1; + index += 1; + } + } + return -1; +} + /** * A secret-ish key (`SECRET_KEY`) and its value: a whole quoted string, or an unquoted run of at least * six characters up to whitespace, a quote, `,`, `;` or `&`. Keys are read in `keys`, which hides each - * URL userinfo's own text: a key there names its username or password + * URL userinfo's username (`UrlUserinfo.ownsUntil`): a key there names the user * (`https://x-access-token:@github.com/o/r`), and its value would run on through the public host and * path. Values are read in `values`, where every earlier mask is one run, as an applied mask would be. * The key scan resumes at each value rather than after it, so a key inside a value is read too: @@ -633,9 +671,8 @@ function visitSecretKeyValues(keys: string, values: string, mask: SecretSpanVisi const valueStart = key.lastIndex; const quote = values[valueStart]; if (quote === '"' || quote === "'") { - const quoted = quote === '"' ? DOUBLE_QUOTED_VALUE : SINGLE_QUOTED_VALUE; - quoted.lastIndex = valueStart; - if (quoted.test(values)) mask(match.index, quoted.lastIndex, '[redacted]'); + const end = quotedValueEnd(values, valueStart); + if (end >= 0) mask(match.index, end, '[redacted]'); continue; } if (delimiterAt < valueStart) { @@ -662,9 +699,9 @@ const HOST_SHAPED_MAX_LENGTH = 260; * `=` assignment follows it; anything else may be a malformed username or password. This is the * same predicate as the LLM diagnostic backstop (`packages/llm/src/scrub-secrets.ts`). For such an * ambiguous userinfo the backstop masks through the authority's end unless a host-shaped name follows - * its last `@`. This projection masks the userinfo and reads the keys and parameters in it from the - * raw ?/# on, or all of them when no `user:password` comes first; their values keep their own extent - * and the rules' ranges merge (`https://gateway?password=P@ss.word1` projects to `https://[redacted]`). + * its last `@`. This projection masks the userinfo and reads every key and parameter in it but its + * username's; their values keep their own extent and the rules' ranges merge + * (`https://gateway?password=P@ss.word1` projects to `https://[redacted]`). * The projection differs from that backstop by design: its authority also ends at `"`, `'`, `,`, * `}` and `]`, so userinfo containing one of them (an already-masked `[REDACTED]` password among * them) is left as it is; it recognizes any scheme of up to 32 characters rather than a finite list; @@ -699,10 +736,11 @@ function visitUrlUserinfo(text: string, mask: SecretSpanVisitor): void { if (query >= 0 && isPublicQueryContact(userinfo, query)) continue; const hasPassword = colon >= 0 && colon < userinfo.length - 1; if (!hasPassword && userinfo.length < 16) continue; - // A raw ?/# leaves the authority ambiguous: from there on the userinfo may also be a query - // another rule reads (`https://gateway?password=P@ss.word1`). A `user:password` before it owns - // its text; with no colon there, nothing marks a username, so the userinfo owns none of it. - const ownsUntil = query < 0 ? at : colon >= 0 && colon < query ? query : 0; + // Only a username owns its text: the part before the first ':' when no raw ?/# comes first. A key + // there names the user (`https://x-access-token:@github.com/o/r` keeps its host). A key in the + // password, after a raw ?/# or in a userinfo with no username is read by the key/value and + // opaque-query rules, which mask its value to its own end (`jdbc:…;password=P@ss;…`). + const ownsUntil = colon >= 0 && (query < 0 || colon < query) ? colon : 0; mask(start, start + at, '[redacted]', { schemeStart: match.index, ownsUntil: start + ownsUntil, @@ -721,9 +759,32 @@ function visitUrlUserinfo(text: string, mask: SecretSpanVisitor): void { * position is redacted, closing that leak path too). Cycle-safe; model/dispatch values stay unchanged. */ export function redactSecretShapedValue(value: unknown): unknown { - return redactSecretShapedWalk(value, new WeakSet()); + return redactSecretShapedWalk(value, new WeakSet(), redactSecretShapedText); } +/** + * The journal projection of a tool's arguments (`redactSecretShapedValue`) and whether every string in it + * settled. A string whose projection still changes on its third pass becomes `[redacted]`, which would give + * different calls one journal identity, so the registry refuses such an effect before it is prepared + * ([effect-journal.md](../../../../docs/reference/shared-core/effect-journal.md) §11). + */ +export function projectSecretShapedValue(value: unknown): { + readonly value: unknown; + readonly settled: boolean; +} { + let settled = true; + const projected = redactSecretShapedWalk(value, new WeakSet(), (text) => { + const fixed = fixedPointOf(redactSecretShapedOnce, text); + if (fixed !== undefined) return fixed; + settled = false; + return '[redacted]'; + }); + return { value: projected, settled }; +} + +/** How the walk projects each string: the display projection, or the journal's settle-checking one. */ +type TextProjection = (text: string) => string; + /** * Does this OBJECT KEY name a secret? The same keyword alternation the `key=value` text rule uses, anchored * as a whole-name test rather than an inline one. @@ -805,7 +866,11 @@ const KEY_QUALIFIERS: ReadonlySet = new Set([ * FOLLOWING a secret-ish flag is redacted; the flag itself is kept, because which flag was passed is * diagnostic and is not the secret. */ -function redactArgVector(items: readonly unknown[], seen: WeakSet): unknown[] { +function redactArgVector( + items: readonly unknown[], + seen: WeakSet, + project: TextProjection, +): unknown[] { const out: unknown[] = []; let redactNext: string | undefined; for (const item of items) { @@ -824,7 +889,7 @@ function redactArgVector(items: readonly unknown[], seen: WeakSet): unkn } if (isSecretishFlag(item)) redactNext = item; } - out.push(redactSecretShapedWalk(item, seen)); + out.push(redactSecretShapedWalk(item, seen, project)); } return out; } @@ -874,9 +939,14 @@ function looksLikePort(value: string): boolean { * remaining text is public. Do not traverse a credential container or carry any partially scrubbed value * into the journal. Only an anchored, known auth scheme may survive as a fixed diagnostic label. */ -function redactUnderKey(key: string, item: unknown, seen: WeakSet): unknown { +function redactUnderKey( + key: string, + item: unknown, + seen: WeakSet, + project: TextProjection, +): unknown { if (!isSecretishKey(key) || item === null || item === undefined) - return redactSecretShapedWalk(item, seen); + return redactSecretShapedWalk(item, seen, project); if (typeof item === 'string') { // Match only the bounded label and the start of a nonempty remainder, never return that remainder. // Canonical labels also keep caller-controlled casing out of the credential projection. @@ -893,13 +963,17 @@ function redactUnderKey(key: string, item: unknown, seen: WeakSet): unkn return '[redacted]'; } -function redactSecretShapedWalk(value: unknown, seen: WeakSet): unknown { - if (typeof value === 'string') return redactSecretShapedText(value); +function redactSecretShapedWalk( + value: unknown, + seen: WeakSet, + project: TextProjection, +): unknown { + if (typeof value === 'string') return project(value); if (typeof value !== 'object' || value === null) return value; if (seen.has(value)) return '[cyclic]'; // break the cycle (mirrors redactInlineMedia) — never re-emit the ref seen.add(value); try { - if (Array.isArray(value)) return redactArgVector(value, seen); + if (Array.isArray(value)) return redactArgVector(value, seen, project); if (!isPlainObject(value)) return value; // Date/RegExp/Map/… — leave for native handling const out: Record = {}; // Scrub the KEY too (a secret-shaped key is redacted; a normal name is unchanged — see the doc above). A @@ -909,8 +983,8 @@ function redactSecretShapedWalk(value: unknown, seen: WeakSet): unknown // inspects strings by shape — so a numeric PIN, a nested credential object, or a high-entropy opaque // token with no recognisable prefix all survive it. `null`/`undefined` are left alone so an absent // field does not become a phantom `[redacted]` in a diagnostic. - Object.defineProperty(out, redactSecretShapedText(key), { - value: redactUnderKey(key, item, seen), + Object.defineProperty(out, project(key), { + value: redactUnderKey(key, item, seen, project), enumerable: true, configurable: true, writable: true, @@ -948,7 +1022,7 @@ function toText(result: unknown): string { function makeSummary(text: string): string { // Recognize LF/TAB-split shapes in the complete materialized text before one-line normalization. Clip only after recognition: a closing quote or URL // authority separator can be beyond the preview. Raw model/spill text stays unchanged. - const recognized = /[\n\t]/.test(text) ? redactSecretShapedOnce(text) : text; + const recognized = /[\n\t]/.test(text) ? redactSecretShapedText(text) : text; const oneLine = redactSecretShapedText(recognized.replace(/\s+/g, ' ').trim()); return oneLine.length <= SUMMARY_MAX ? oneLine : `${oneLine.slice(0, SUMMARY_MAX)}…`; } diff --git a/packages/core/src/tools/fixed-point.test.ts b/packages/core/src/tools/fixed-point.test.ts index 9e601bf6..df9ac902 100644 --- a/packages/core/src/tools/fixed-point.test.ts +++ b/packages/core/src/tools/fixed-point.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { toFixedPoint } from './fixed-point.js'; +import { fixedPointOf, toFixedPoint } from './fixed-point.js'; describe('toFixedPoint', () => { const failed = '[failed]'; @@ -37,3 +37,17 @@ describe('toFixedPoint', () => { expect(calls).toEqual(['a', 'b', 'c']); }); }); + +describe('fixedPointOf', () => { + it('reports a settled output and an unsettled one apart', () => { + let calls = 0; + const settles = (text: string) => (text === 'a' ? 'b' : text); + const drifts = (text: string) => { + calls += 1; + return `${text}+`; + }; + expect(fixedPointOf(settles, 'a')).toBe('b'); + expect(fixedPointOf(drifts, 'a')).toBeUndefined(); + expect(calls).toBe(3); + }); +}); diff --git a/packages/core/src/tools/fixed-point.ts b/packages/core/src/tools/fixed-point.ts index 57cd3b5d..526c5185 100644 --- a/packages/core/src/tools/fixed-point.ts +++ b/packages/core/src/tools/fixed-point.ts @@ -1,13 +1,21 @@ /** * Repeats one projection pass until its output stops changing, at most three passes. A pass can - * change the context the next pass decides on, so a settled output is a fixed point, and an output - * that still changes on the third pass fails closed to `failed`. Callers choose a `failed` value the - * pass leaves unchanged, so the result is stable under another call either way. + * change the context the next pass decides on, so a settled output is a fixed point; an output that + * still changes on the third pass has not settled, and this returns `undefined` for it. */ -export function toFixedPoint(pass: (text: string) => string, text: string, failed: string): string { +export function fixedPointOf(pass: (text: string) => string, text: string): string | undefined { const once = pass(text); if (once === text) return once; const twice = pass(once); if (twice === once) return once; - return pass(twice) === twice ? twice : failed; + return pass(twice) === twice ? twice : undefined; +} + +/** + * The pass's fixed point (`fixedPointOf`), failing closed to `failed` when it has not settled. Callers + * choose a `failed` value the pass leaves unchanged, so the result is stable under another call + * either way. + */ +export function toFixedPoint(pass: (text: string) => string, text: string, failed: string): string { + return fixedPointOf(pass, text) ?? failed; } diff --git a/packages/core/src/tools/registry.test.ts b/packages/core/src/tools/registry.test.ts index 4671b574..79451f1d 100644 --- a/packages/core/src/tools/registry.test.ts +++ b/packages/core/src/tools/registry.test.ts @@ -1,6 +1,8 @@ import type { AbortSignalLike, Scope, ToolPolicy } from '@relavium/shared'; import { describe, expect, it, vi } from 'vitest'; +import { z } from 'zod'; +import { projectSecretShapedValue } from './bounding.js'; import { BUILTIN_TOOLS, BUILTIN_TOOL_IDS } from './builtins.js'; import { ToolArgsInvalidError, @@ -21,11 +23,19 @@ import type { ToolApprovalDecision, ToolApprovalRequest, ToolCallPart, + ToolDef, ToolDispatchContext, ToolHost, } from './types.js'; import { createInMemoryEffectJournal } from '../engine/execution-host.js'; +// Observe the journal projection, retaining its actual implementation, so one case can force a value +// that does not settle: real inputs that never settle are no longer known. +vi.mock('./bounding.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, projectSecretShapedValue: vi.fn(actual.projectSecretShapedValue) }; +}); + /* --- helpers --- */ function call(name: string, args?: unknown, providerExecuted?: boolean): ToolCallPart { @@ -1653,3 +1663,101 @@ describe('ToolRegistry — remaining hardening regressions (TG-2, TG-4)', () => expect(err.reason).toBe('insecure_url'); }); }); + +describe('ToolRegistry — journal identity of credential-shaped arguments (PR95 round 7)', () => { + it('refuses a tiered effect whose journal projection does not settle, before preparing it', async () => { + const actual = await vi.importActual('./bounding.js'); + vi.mocked(projectSecretShapedValue).mockImplementation((value) => ({ + value: actual.projectSecretShapedValue(value).value, + settled: !JSON.stringify(value).includes('UNSETTLED'), + })); + try { + const writeFile = vi.fn['writeFile']>((path, data) => + Promise.resolve({ path, bytesWritten: data.length }), + ); + const host = stubHost(); + if (host.fs === undefined) throw new Error('missing stub fs'); + const context = ctx(); + const prepare = vi.fn(context.effects.prepare); + const err: unknown = await registry(stubHost({ fs: { ...host.fs, writeFile } })) + .dispatch( + call('write_file', { path: './out.txt', content: 'UNSETTLED body', append: true }), + { + ...context, + effects: { ...context.effects, prepare }, + }, + ) + .catch((error: unknown) => error); + expect(err, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').toBeInstanceOf(ToolArgsInvalidError); + if (!(err instanceof ToolArgsInvalidError)) throw new Error('missing refusal'); + expect(err.fields, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').toEqual(['content']); + expect(err.message.includes('UNSETTLED'), 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').toBe(false); + expect(prepare, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').not.toHaveBeenCalled(); + expect(writeFile, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').not.toHaveBeenCalled(); + } finally { + vi.mocked(projectSecretShapedValue).mockImplementation(actual.projectSecretShapedValue); + } + }); + + it('names the arguments it refuses through the same masker', async () => { + const actual = await vi.importActual('./bounding.js'); + vi.mocked(projectSecretShapedValue).mockImplementation((value) => ({ + value: actual.projectSecretShapedValue(value).value, + settled: !JSON.stringify(value).includes('UNSETTLED'), + })); + try { + // An MCP tool's open schema lets the model choose the argument names themselves. + const dispatch = vi.fn(() => Promise.resolve({ receipt: 'published' })); + const tool: ToolDef = { + id: 'open_publish', + source: 'mcp', + description: 'Offline open-schema fixture', + llmVisibleParams: { type: 'object' }, + parseArgs: (raw) => z.record(z.string(), z.unknown()).parse(raw), + policy: { fsScoped: false, spawnsProcess: false, requiresGateApproval: false }, + effect: () => 3, + dispatch, + }; + const context = ctx({ grantedToolIds: new Set([tool.id]) }); + const prepare = vi.fn(context.effects.prepare); + const err: unknown = await createToolRegistry({ tools: [tool], host: stubHost() }) + .dispatch( + call('open_publish', { + content: 'UNSETTLED body', + ghp_abcdefghijklmnopqrstuvwxyz0123UNSETTLED: 'x', + 'sk-proj-abcdefghijklmnopqrstuvUNSETTLED': 'y', + }), + { ...context, effects: { ...context.effects, prepare } }, + ) + .catch((error: unknown) => error); + expect(err, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').toBeInstanceOf(ToolArgsInvalidError); + if (!(err instanceof ToolArgsInvalidError)) throw new Error('missing refusal'); + expect(err.fields, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').toEqual(['content', '[redacted]']); + for (const raw of ['ghp_', 'sk-proj', 'UNSETTLED']) { + expect(err.message.includes(raw), 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').toBe(false); + } + expect(prepare, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').not.toHaveBeenCalled(); + expect(dispatch, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').not.toHaveBeenCalled(); + } finally { + vi.mocked(projectSecretShapedValue).mockImplementation(actual.projectSecretShapedValue); + } + }); + + it('projects an exactly admitted-size quoted write after the write lands', async () => { + const size = 8 * 1024 * 1024; + const writeFile = vi.fn['writeFile']>((path, data) => + Promise.resolve({ path, bytesWritten: data.length }), + ); + const host = stubHost(); + if (host.fs === undefined) throw new Error('missing stub fs'); + const out = await registry(stubHost({ fs: { ...host.fs, writeFile } })).dispatch( + call('write_file', { path: './out.txt', content: `password="${'a'.repeat(size - 11)}"` }), + ctx(), + ); + expect(writeFile, 'SECRETS_B7_CORE_QUOTED_SCAN').toHaveBeenCalledTimes(1); + expect(out.events.call.toolInput, 'SECRETS_B7_CORE_QUOTED_SCAN').toEqual({ + path: './out.txt', + content: '[redacted]', + }); + }, 15_000); +}); diff --git a/packages/core/src/tools/registry.ts b/packages/core/src/tools/registry.ts index 51454c0b..06d8b6f8 100644 --- a/packages/core/src/tools/registry.ts +++ b/packages/core/src/tools/registry.ts @@ -19,6 +19,7 @@ import { import { boundForModel, + projectSecretShapedValue, redactInlineMedia, redactSecretShapedText, redactSecretShapedValue, @@ -338,13 +339,23 @@ async function prepareEffect( effective: Readonly>, ctx: ToolDispatchContext, ): Promise { - try { - const args = [ - ctx.effectSlot, + const journal = projectInput(def, effective, ctx.secretArgKeys, 'journal'); + if (journal.unsettled.length > 0) { + // A value whose redacted projection never settles projects to `[redacted]`, which any other such + // value shares: its digest could replay one call's result for a different call (§4), so the + // effect is refused before anything is journaled. The model can correct the argument. An open + // schema lets the model choose an argument's name, so the names are masked like any other text. + const fields = [...new Set(journal.unsettled.map(redactSecretShapedText))]; + throw new ToolArgsInvalidError( def.id, - tier, - sanitizeInput(def, effective, ctx.secretArgKeys, 'journal'), - ] as const; + fields, + `tool \`${def.id}\`: credential-shaped text in ${fields.join( + ', ', + )} has no stable redacted form, so this effect cannot be given a journal identity`, + ); + } + try { + const args = [ctx.effectSlot, def.id, tier, journal.input] as const; return await (ctx.effectAttempt === undefined ? ctx.effects.prepare(...args) : ctx.effects.prepare(...args, undefined, ctx.effectAttempt)); @@ -1061,6 +1072,19 @@ function sanitizeInput( secretArgKeys: ReadonlySet | undefined, purpose: 'display' | 'journal', ): Readonly> { + return projectInput(def, effective, secretArgKeys, purpose).input; +} + +/** + * `sanitizeInput`'s projection, with the top-level arguments whose journal projection did not settle + * (always none for the display projection, which needs no identity). + */ +function projectInput( + def: ToolDef, + effective: Readonly>, + secretArgKeys: ReadonlySet | undefined, + purpose: 'display' | 'journal', +): { readonly input: Readonly>; readonly unsettled: readonly string[] } { const out: Record = { ...effective }; for (const key of def.configOnlyParams ?? []) { delete out[key]; @@ -1094,12 +1118,27 @@ function sanitizeInput( // A display preview may omit a tail; the journal's collision guard may not. Both still exclude // declared secrets and credential/media-shaped values BEFORE the host computes its digest. // Recognition needs the complete value: clipping may remove a quote or URL authority separator. - const walked = redactSecretShapedValue(redactInlineMedia(out)); + const media = redactInlineMedia(out); + const journal = purpose === 'journal' ? projectSecretShapedValue(media) : undefined; + const walked = journal === undefined ? redactSecretShapedValue(media) : journal.value; const projection = purpose === 'display' ? boundArgsForDisplay(walked) : walked; + // Name the arguments that did not settle (the walk itself reports only that one did not). + const unsettled = + journal === undefined || journal.settled || !isPlainRecord(media) + ? [] + : Object.keys(media) + .filter((key) => !projectSecretShapedValue({ [key]: media[key] }).settled) + .toSorted(byCodeUnit); // The walk preserves plain-object shape, so this is the same record with scrubbed members; the guard is // a boundary check rather than a cast, and the fallback can only be reached if the walk ever stops // returning an object for an object input. - return isPlainRecord(projection) ? projection : out; + return { + input: isPlainRecord(projection) ? projection : out, + unsettled: + journal?.settled === false && unsettled.length === 0 + ? Object.keys(out).toSorted(byCodeUnit) + : unsettled, + }; } /** diff --git a/packages/llm/src/scrub-secrets.test.ts b/packages/llm/src/scrub-secrets.test.ts index 6447619b..19962875 100644 --- a/packages/llm/src/scrub-secrets.test.ts +++ b/packages/llm/src/scrub-secrets.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it, vi } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { toFixedPoint } from './fixed-point.js'; import { scrubSecrets as rootScrubSecrets } from './index.js'; @@ -12,6 +12,11 @@ vi.mock('./fixed-point.js', async (importOriginal) => { return { ...actual, toFixedPoint: vi.fn(actual.toFixedPoint) }; }); +// The observation must not keep admitted-size inputs and outputs alive for the whole file. +afterEach(() => { + vi.mocked(toFixedPoint).mockClear(); +}); + // Synthetic credentials assembled from fragments, never real provider material. const key = ['sk-', 'ant-', 'api03-', 'AbCdEf0123456789xyz'].join(''); const googleKey = ['AI', 'za', 'SyA1234567890abcdefghijklmnopqrstuv'].join(''); @@ -1324,6 +1329,18 @@ describe('PR95 review round 6: the label bound behind a terminal sequence, fragm } }); + it('fails closed on nine labels that only the normalized view joins', () => { + const bound = (value: unknown) => expect(value, 'SECRETS_B4_LABEL_RUN_BOUND'); + // A terminal sequence normalization removes splits the raw run into four and five labels. + for (const label of ['Bearer', 'Basic']) { + for (const lead of ['Authorization: Bearer ', 'x-api-key: ', '']) { + bound( + scrubSecrets(`${lead}${label.repeat(4)}\u001b[0m${label.repeat(5)} opaqueTOKEN123`), + ).toBe('[REDACTED]'); + } + } + }); + it('masks a literal secret query name in a fragment', () => { const fragment = (value: unknown) => expect(value, 'SECRETS_B6_QUERY_FRAGMENT'); fragment(scrubSecrets('https://[::1]#id_token=Zk81Pq0aLm3nB7xY@127.0.0.1/cb')).toBe( diff --git a/tools/causal-controls/README.md b/tools/causal-controls/README.md index c46dbe90..cedf34ef 100644 --- a/tools/causal-controls/README.md +++ b/tools/causal-controls/README.md @@ -128,6 +128,13 @@ scratch tree through a symlink. The bound includes the final export record: at m failure. Selected runs never claim complete-matrix acceptance. Source HEAD and working-tree capture attribution describe the execution actually performed. +`--shard /` deals the declared controls round-robin in declaration +order, so the `count` shards partition the family; each shard is a selection with +`complete: false` and its own export, and records `shard: { index, count, declared, +members }`, so the shards' summaries can be joined against the declaration. Only every +shard of one count passing, each within the export bounds, covers the family. The +secrets family runs as two shards because one export of all its controls exceeds the +64-MiB bound; a full unsharded run still executes every control, but it cannot export. The permanent coordinator integration guard runs three bounded actual captures. Each family proves synthetic fixed/red/restored execution before a real assertion diff --git a/tools/causal-controls/provenance.mjs b/tools/causal-controls/provenance.mjs index a90c5876..0bdcb3e6 100644 --- a/tools/causal-controls/provenance.mjs +++ b/tools/causal-controls/provenance.mjs @@ -13,8 +13,9 @@ import { digest } from '../usage-coverage-controls/source-graph.mjs'; export function selectControls(mutations, requested) { assert.ok( - requested.length === 0 || (requested.length === 2 && requested[0] === '--mechanism'), - 'Usage: check.mjs [--mechanism declared-id]', + requested.length === 0 || + (requested.length === 2 && ['--mechanism', '--shard'].includes(requested[0])), + 'Usage: check.mjs [--mechanism declared-id | --shard index/count]', ); assert.ok( mutations.length > 0 && new Set(mutations.map((entry) => entry.id)).size === mutations.length, @@ -28,6 +29,12 @@ export function selectControls(mutations, requested) { 'Empty declared inventory', ); } + if (requested[0] === '--shard') { + const { index, count } = parseShard(requested[1]); + assert.ok(count <= mutations.length, 'Shard out of range'); + // Declaration order dealt round-robin: every control belongs to exactly one of the shards. + return mutations.filter((_, position) => position % count === index - 1); + } const selected = requested.length ? mutations.filter((entry) => entry.id === requested[1]) : mutations; @@ -35,6 +42,16 @@ export function selectControls(mutations, requested) { return selected; } +/** `index/count`, 1-based, as `--shard` takes it. */ +export function parseShard(value) { + const match = /^([1-9]\d{0,2})\/([1-9]\d{0,2})$/.exec(value ?? ''); + assert.ok(match, 'Usage: --shard index/count'); + const index = Number(match[1]); + const count = Number(match[2]); + assert.ok(index <= count, 'Shard out of range'); + return { index, count }; +} + export function ownedTools(repository, roots) { const files = []; function walk(path) { diff --git a/tools/causal-controls/provenance.test.mjs b/tools/causal-controls/provenance.test.mjs index fd47946b..951565c7 100644 --- a/tools/causal-controls/provenance.test.mjs +++ b/tools/causal-controls/provenance.test.mjs @@ -154,6 +154,21 @@ test('native child and target need same pinned child PID, parent link and depend assert.throws(() => assertEvaluations(candidate, { files }, target.path, graph, options)); }); }); +test('shards deal the declaration round-robin into disjoint, complete parts', () => { + const entries = ['a', 'b', 'c', 'd', 'e'].map((id) => ({ + id, + before: 'old', + after: 'new', + expectedTests: 1, + })); + const ids = (args) => selectControls(entries, args).map((entry) => entry.id); + assert.deepEqual(ids(['--shard', '1/2']), ['a', 'c', 'e']); + assert.deepEqual(ids(['--shard', '2/2']), ['b', 'd']); + assert.deepEqual(ids(['--shard', '1/1']), ['a', 'b', 'c', 'd', 'e']); + for (const shard of ['0/2', '3/2', '1/6', '1/0', '01/2', '1/2/3', 'x/2', '', '-1/2']) + assert.throws(() => selectControls(entries, ['--shard', shard])); + assert.throws(() => selectControls(entries, ['--shard'])); +}); test('unknown, ambiguous, empty and ineffective selection refuse before work', () => { const entry = { id: 'declared', before: 'old', after: 'new', expectedTests: 1 }; assert.deepEqual(selectControls([entry], ['--mechanism', 'declared']), [entry]); @@ -267,10 +282,10 @@ test('every declared filesystem and secrets anchor matches current production so }); test('secrets workload corrections retain independent complete-count and reader-witness pins', () => { - assert.equal(secretsControls.length, 110); + assert.equal(secretsControls.length, 120); assert.equal( secretsControls.reduce((count, control) => count + 1 + (control.witnesses ?? []).length, 0), - 491, + 501, ); for (const [id, file, expectedTests] of [ ['run-default-secret-reader', 'apps/cli/src/commands/run.test.ts', 84], @@ -323,7 +338,16 @@ test('secrets workload corrections retain independent complete-count and reader- 'core-key-value-through-masks', 'core-scheme-nested-label', 'core-mask-boundaries', - ].map((id) => [id, ['packages/core/src/tools/bounding.test.ts'], 185]), + // Round 7: userinfo ownership, the quoted-value scan, chaining, the summary and the journal. + 'core-userinfo-username-zone', + 'core-split-zone-cap', + 'core-quoted-value-scan', + 'core-standalone-chain', + 'core-glued-span-boundary', + 'core-summary-fixed-point', + 'core-journal-settled', + ].map((id) => [id, ['packages/core/src/tools/bounding.test.ts'], 195]), + ['core-unsettled-refusal', ['packages/core/src/tools/registry.test.ts'], 112], ...[ 'diagnostic-joined-before-masking', 'diagnostic-joined-original-boundary', @@ -370,7 +394,9 @@ test('secrets workload corrections retain independent complete-count and reader- 'diagnostic-userinfo-authority-rest', 'diagnostic-userinfo-rest-length', 'diagnostic-url-split-retry', - ].map((id) => [id, ['packages/llm/src/scrub-secrets.test.ts'], 218]), + 'diagnostic-label-run-normalized', + 'diagnostic-label-run-raw', + ].map((id) => [id, ['packages/llm/src/scrub-secrets.test.ts'], 219]), ...['diagnostic-inline-projection-order', 'diagnostic-original-control-boundary'].map((id) => [ id, ['apps/cli/src/render/sanitize.test.ts', 'apps/cli/src/process/render-error.test.ts'], diff --git a/tools/causal-controls/run.mjs b/tools/causal-controls/run.mjs index d03ae37c..0dd6d693 100644 --- a/tools/causal-controls/run.mjs +++ b/tools/causal-controls/run.mjs @@ -36,6 +36,7 @@ import { mergeFiles, assertEvaluations, selectControls, + parseShard, phaseEnvironment, exactReplacement, assertSelectedAnchors, @@ -53,12 +54,23 @@ export async function runControls({ repository, evidenceKind, mutations, toolRoo const vitest = join(dirname(require.resolve('vitest/package.json')), 'vitest.mjs'); const requested = process.argv.slice(2); const selected = selectControls(mutations, requested); + // A shard is a selection like any other, so it never claims the complete family; it records which + // controls it was dealt, so the shards' summaries can be joined against the declaration. + const shard = + requested[0] === '--shard' + ? { + ...parseShard(requested[1]), + declared: mutations.length, + members: selected.map((entry) => entry.id), + } + : undefined; const scratch = realpathSync(mkdtempSync(join(tmpdir(), `relavium-${evidenceKind}-controls-`))); const graph = join(scratch, 'graph'); const summary = { version: 1, complete: false, node: process.version, + ...(shard === undefined ? {} : { shard }), mechanisms: [], restorations: [], }; diff --git a/tools/ci/require-w8-acceptance.test.mjs b/tools/ci/require-w8-acceptance.test.mjs index 2cb1441a..0365f319 100644 --- a/tools/ci/require-w8-acceptance.test.mjs +++ b/tools/ci/require-w8-acceptance.test.mjs @@ -94,16 +94,21 @@ test('the actual required workflow joins both full offline controls and checks t assert.equal(job['continue-on-error'], undefined); assert.equal(job.if, undefined); assert.equal(job['timeout-minutes'], 60); + // Two legs, each running and exporting a disjoint half; neither may stop the other early. + assert.deepEqual(job.strategy, { 'fail-fast': false, matrix: { shard: [1, 2] } }); assert.ok(job.steps.some((step) => step.run === 'pnpm install --frozen-lockfile')); assert.ok(job.steps.some((step) => step.run === 'pnpm test:acceptance-tools')); assert.ok(job.steps.some((step) => step.with?.['node-version-file'] === '.nvmrc')); - const command = job.steps.find((step) => step.run === `pnpm check:${kind}-mutations`); + const command = job.steps.find( + (step) => step.run === `pnpm check:${kind}-mutations --shard \${{ matrix.shard }}/2`, + ); assert.equal(command['continue-on-error'], undefined); assert.equal(command.if, undefined); assert.equal(command.env.RELAVIUM_ACCEPTANCE_EVIDENCE, '${{ runner.temp }}/wave2-acceptance'); const artifact = job.steps.find((step) => step.uses?.startsWith('actions/upload-artifact@')); assert.equal(artifact.if, '${{ always() }}'); assert.equal(artifact.with.path, '${{ runner.temp }}/wave2-acceptance/secrets'); + assert.match(artifact.with.name, /^wave2-secrets-\$\{\{ matrix\.shard \}\}-of-2-/); assert.equal(artifact.with['retention-days'], 7); } const filesystem = workflow.jobs['fs-mutations']; diff --git a/tools/secrets-controls/README.md b/tools/secrets-controls/README.md index 9970385a..ace38bc4 100644 --- a/tools/secrets-controls/README.md +++ b/tools/secrets-controls/README.md @@ -4,7 +4,7 @@ Step 4 first-run input matrix, Step 5 snapshot/export, Step 7 provider-header and PR #95 B2 redaction controls on physical source copies. `--mechanism ` supports diagnosis only; selected runs retain `complete: false`. The current manifest -declares **110 independent controls and 491 required named assertion witnesses**. +declares **120 independent controls and 501 required named assertion witnesses**. These counts describe the workload; a declaration is not evidence that its removal passed. Actual fixed/removal/restored results belong to the coordinator's summary. @@ -178,8 +178,9 @@ modern `sk-` body characters, joined-span application, the original-text left-bo check, split scheme labels, split API-header labels and raw `?`/`#` password recognition independently. Four LLM controls remove the before-masking LF/TAB probe, its original left-boundary check, split API-header recognition and raw-password recognition. -Every core phase executes the complete 185-case bounding file; every new LLM phase -executes the complete 218-case scrubber file. Independent ordinary tooling tests pin +Every core projection phase executes the complete 195-case bounding file (the journal-identity +control's phases the complete 112-case registry file); every new LLM phase executes the complete +219-case scrubber file. Independent ordinary tooling tests pin these inventories, the 278-case overlap cohort, 180-case CLI diagnostic cohort and 18-case command-export cohort. Actual collection still has to match in every phase. @@ -259,16 +260,19 @@ The LLM original-boundary removal now removes both the numeric coarse previous-word prefilter and the rule's own boundary predicate; retaining either would leave the tested policy active. Header/key recovery removals still change their original-rule boundary only, and all removals retain the same canonical -matchers and bounded raw/joined standalone visitors. Current collection is 25 -complete files / 1,494 cases, with each control's exact workload and named target +matchers and bounded raw/joined standalone visitors. Current collection is 26 +complete files / 1,617 cases, with each control's exact workload and named target checked separately. These are updated declarations and author checks, not new physical causal captures or a clean review round. R3 is preserved as NONCLEAN; later corrective acceptance belongs to the owning phase register. Prior selected `7e9096a8` captures retain their older source/workload attribution. -The current 110-control / 330-phase full declaration projects 1,987 retained files -(110 × 18 + 7) under the unchanged 2,000-file / 64-MiB export bounds. This is a capacity -calculation, not a measured full capture. Historical 41-control captures retain their +The current 120-control / 360-phase full declaration runs as two round-robin shards of 60 +controls (`--shard 1/2` and `--shard 2/2`), each exporting its own evidence: each projects 1,087 +retained files (60 × 18 + 7) of the unchanged 2,000. Locally both exports held exactly that many +and measured 45.50 MiB and 45.48 MiB, final export record included, against the unchanged 64-MiB +bound. That is one local measurement at the round-7 source, not a CI capture; one export of the +whole family would exceed the byte bound. Historical 41-control captures retain their original source and cohort attribution; selected B2 or original-boundary recovery captures are not full-family acceptance. @@ -383,6 +387,36 @@ separator right before it; that only spares reporting the main view's range a se output cannot show it and it has neither a test nor a control. The shared END search is pinned by the admitted-size case deadline alone, like round 4's deadline-only guards. +The seventh round found that a key in a URL userinfo's own text could still hide a real +credential. An unambiguous userinfo owned every key in it, so a key in its password or parameter +position (`jdbc:sqlserver://db:1433;user=sa;password=P@ss…`, `https://svc;a=1;password=P@ss…`) and, +in the LF/TAB-joined view, a key on the next line (`DATABASE_URL=https://h\nDB_PASSWORD=P@ss…`) left +its value's text after the last `@` visible. Only a userinfo's username — the text before its first +`:` when no raw `?`/`#` comes before that colon — now owns its keys, and never past an LF or TAB. The +quoted-value pattern's repeated group exhausted V8's RegExp stack at an exactly admitted 8-MiB value, +after a write had landed; quoted values are now found by a character scan. A credential glued to the +end of a fixed-length one (`AKIA…ASIA…`) is a credential of its own, so a run of them is masked in one +pass, and a split span glued to the one before it is bounded by it as by an applied mask. Summaries +recognize split shapes with the projection's fixed point. A projection that does not settle becomes +one constant, which would give two different calls one journal digest, so a tiered call whose +journal projection does not settle is refused before its prepare, its argument names masked. The +family also could not export its evidence: one export of its 110 controls measured about 77 MiB +against the unchanged 64-MiB bound, and no author gate exercised the export, because only the +dedicated job sets `RELAVIUM_ACCEPTANCE_EVIDENCE`. It now runs as two shards +([causal-controls](../causal-controls/README.md)), and both ran locally with the export enabled. +Ten controls were added: `core-userinfo-username-zone`, `core-split-zone-cap`, +`core-quoted-value-scan`, `core-standalone-chain`, `core-glued-span-boundary`, +`core-summary-fixed-point`, `core-journal-settled` and `core-unsettled-refusal`, and +`diagnostic-label-run-normalized` and `diagnostic-label-run-raw`, which remove one half of the +nine-label bound each, since removing the whole line (`diagnostic-label-run-bound`, kept) left the +normalized half untested. `core-userinfo-query-zone` is re-anchored and no control is retired. +Pinned by tests without a control: the username zone's colon comparisons, the merged run's +replacement, the fill's merge and order and the place of opaque values among the masks, the key +rule's keyword alternatives, suffix and boundary and the value's delimiters, the quoted scan's escape +and line-break rules, the refused argument names' masking and the helper's settled report. The header +label join's `:` prefilter only skips work whose result is the text unchanged, so the output cannot +show it and it has neither a test nor a control. + The thin entry point uses the same coordinator, outcome and bounded process join as the filesystem family in [causal-controls](../causal-controls/README.md). The provider-header diagnostic control pins the store's fixed throwable message diff --git a/tools/secrets-controls/mutations.mjs b/tools/secrets-controls/mutations.mjs index e2c4aa49..5855b51f 100644 --- a/tools/secrets-controls/mutations.mjs +++ b/tools/secrets-controls/mutations.mjs @@ -150,6 +150,8 @@ const hostFollowsLine = ' return host.length <= HOST_SHAPED_MAX_LENGTH && HOST_SHAPED_AUTHORITY.test(host);'; const llmFixedPoint = ' return toFixedPoint(scrubOnce, text, MASKED);'; const coreFixedPoint = " return toFixedPoint(redactSecretShapedOnce, text, '[redacted]');"; +const coreUserinfoZone = + ' const ownsUntil = colon >= 0 && (query < 0 || colon < query) ? colon : 0;'; const headerTokenMask = ' mask: tokenGroup,\n joinedPattern:'; const stackedLabelCase = 'masks a header token after up to eight stacked labels and fails closed beyond them'; @@ -252,6 +254,16 @@ const coreOriginalBoundary = String.raw` const candidates = /(^|[^\w])([A-Za-z] if (match === null) continue; matchedUntil = JOINED_STANDALONE_CREDENTIAL.lastIndex; mask(start, matchedUntil, '[redacted]'); + // A credential glued to the end of this one starts one of its own, as after an applied mask, so a + // run of fixed-length ids (AKIA…ASIA…) is masked whole in one pass. + for (let glued = matchedUntil; /[A-Za-z]/.test(joined[glued] ?? ''); glued = matchedUntil) { + JOINED_STANDALONE_CREDENTIAL.lastIndex = 0; + if (!JOINED_STANDALONE_CREDENTIAL.test(preview(glued))) break; + JOINED_STANDALONE_CREDENTIAL.lastIndex = glued; + if (JOINED_STANDALONE_CREDENTIAL.exec(joined) === null) break; + matchedUntil = JOINED_STANDALONE_CREDENTIAL.lastIndex; + mask(glued, matchedUntil, '[redacted]'); + } } } @@ -318,21 +330,26 @@ function visitSplitSpans( } return target + removed; } + let previousEnd = -1; visit(joined, (start, end, replacement, url) => { // A separator right before a URL's scheme is outside the URL, so it splits nothing. const schemeStart = url === undefined ? undefined : advance(url.schemeStart, true); const originalStart = advance(start, !ownsEdges); const ownsUntil = url === undefined ? undefined : advance(url.ownsUntil, ownsEdges); const originalEnd = advance(end, ownsEdges); - if (boundary?.test(text[originalStart - 1] ?? '')) return;`; + // A span that starts where the visitor's previous one ended is glued to a credential, which bounds + // it as an applied mask would. + const glued = originalStart === previousEnd; + previousEnd = originalEnd; + if (!glued && boundary?.test(text[originalStart - 1] ?? '')) return;`; const withoutCoreOriginalBoundary = coreOriginalBoundary .replace( String.raw` const candidates = /(^|[^\w])([A-Za-z])/g;`, String.raw` const candidates = /([A-Za-z])/g;`, ) - .replace(" if (boundary?.test(text[originalStart - 1] ?? '')) return;", ''); + .replace(" if (!glued && boundary?.test(text[originalStart - 1] ?? '')) return;", ''); /** The complete core projection test file every core projection control runs. */ -const CORE_PROJECTION_CASES = 185; +const CORE_PROJECTION_CASES = 195; function coreProjectionControl(id, before, after, targets, marker = coreProjectionMarker) { return control( id, @@ -346,8 +363,10 @@ function coreProjectionControl(id, before, after, targets, marker = coreProjecti targets.slice(1).map((target) => ({ target, marker })), ); } +/** The complete registry test file the journal-identity control runs. */ +const REGISTRY_CASES = 112; /** The complete scrubber test file every LLM diagnostic control runs. */ -const LLM_SCRUB_CASES = 218; +const LLM_SCRUB_CASES = 219; function diagnosticControl(id, before, after, targets, marker = diagnosticMarker) { return control( id, @@ -1177,7 +1196,28 @@ export const mutations = [ ['masks a quoted standalone token after either label'], 'SECRETS_B4_LABEL_LEAD', ), - // Nine concatenated labels exceed every bounded run, so the diagnostic fails closed. + // Nine concatenated labels exceed every bounded run, so the diagnostic fails closed, read in the + // normalized view and in the raw text alike (round 7: each half has its own control). + ...[ + [ + 'diagnostic-label-run-normalized', + ' if (LABEL_RUN_PAST_BOUND.test(text)) return MASKED;', + 'fails closed on nine labels that only the normalized view joins', + ], + [ + 'diagnostic-label-run-raw', + ' if (LABEL_RUN_PAST_BOUND.test(normalized)) return MASKED;', + 'fails closed on nine labels behind a terminal sequence', + ], + ].map(([id, after, target]) => + diagnosticControl( + id, + ' if (LABEL_RUN_PAST_BOUND.test(normalized) || LABEL_RUN_PAST_BOUND.test(text)) return MASKED;', + after, + [target], + 'SECRETS_B4_LABEL_RUN_BOUND', + ), + ), diagnosticControl( 'diagnostic-label-run-bound', ' if (LABEL_RUN_PAST_BOUND.test(normalized) || LABEL_RUN_PAST_BOUND.test(text)) return MASKED;', @@ -1446,7 +1486,7 @@ export const mutations = [ ), coreProjectionControl( 'core-userinfo-query-zone', - ' const ownsUntil = query < 0 ? at : colon >= 0 && colon < query ? query : 0;', + coreUserinfoZone, ' const ownsUntil = at;', [ 'masks an ambiguous userinfo with the query values its later rules would mask whole', @@ -1508,6 +1548,70 @@ export const mutations = [ ), 'SECRETS_B4_CORE_SPLIT_EDGE', ), + // Round 7: only a username owns its userinfo's text, and none past an LF or TAB; quoted values are + // scanned without a RegExp repeat; glued credentials chain; the summary uses the projection's fixed + // point; and an effect whose journal projection never settles is refused before it is prepared. + coreProjectionControl( + 'core-userinfo-username-zone', + coreUserinfoZone, + ' const ownsUntil = query < 0 ? at : colon >= 0 && colon < query ? query : 0;', + ["reads a key in a userinfo's password or parameter position"], + 'SECRETS_B7_CORE_USERNAME_ZONE', + ), + coreProjectionControl( + 'core-split-zone-cap', + ' : { schemeStart, ownsUntil: split < 0 ? ownsUntil : originalStart + split },', + ' : { schemeStart, ownsUntil },', + ["owns no text past a split userinfo's first LF or TAB"], + 'SECRETS_B7_CORE_SPLIT_ZONE', + ), + coreProjectionControl( + 'core-quoted-value-scan', + ' const end = quotedValueEnd(values, valueStart);', + String.raw` const quoted = quote === '"' ? /"(?:[^"\\\r\n]|\\[^\r\n])*"/y : /'(?:[^'\\\r\n]|\\[^\r\n])*'/y; + quoted.lastIndex = valueStart; + const end = quoted.test(values) ? quoted.lastIndex : -1;`, + ['projects exactly admitted-size quoted values without exhausting the RegExp stack'], + 'SECRETS_B7_CORE_QUOTED_SCAN', + ), + coreProjectionControl( + 'core-standalone-chain', + " for (let glued = matchedUntil; /[A-Za-z]/.test(joined[glued] ?? ''); glued = matchedUntil) {", + ' for (let glued = matchedUntil; false; glued = matchedUntil) {', + ['masks a run of glued fixed-length ids in one pass, split or not'], + 'SECRETS_B7_CORE_CHAIN', + ), + coreProjectionControl( + 'core-glued-span-boundary', + ' const glued = originalStart === previousEnd;', + ' const glued = false;', + ['masks a run of glued fixed-length ids in one pass, split or not'], + 'SECRETS_B7_CORE_CHAIN', + ), + coreProjectionControl( + 'core-summary-fixed-point', + String.raw` const recognized = /[\n\t]/.test(text) ? redactSecretShapedText(text) : text;`, + String.raw` const recognized = /[\n\t]/.test(text) ? redactSecretShapedOnce(text) : text;`, + ["summarizes with the projection's fixed point before one-line normalization"], + 'SECRETS_B7_CORE_SUMMARY', + ), + coreProjectionControl( + 'core-journal-settled', + ' settled = false;', + '', + ['reports whether a journal projection settled'], + 'SECRETS_B7_CORE_JOURNAL_SETTLED', + ), + control( + 'core-unsettled-refusal', + 'packages/core/src/tools/registry.ts', + ' if (journal.unsettled.length > 0) {', + ' if (false) {', + ['packages/core/src/tools/registry.test.ts'], + REGISTRY_CASES, + 'refuses a tiered effect whose journal projection does not settle, before preparing it', + 'SECRETS_B7_CORE_UNSETTLED_REFUSAL', + ), coreProjectionControl( 'core-fixed-point', coreFixedPoint, From 6f7dd3426a946e22f39bb8d81e4798de451810a5 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Wed, 21 Oct 2026 16:45:25 +0300 Subject: [PATCH 17/97] fix(core): address the eighth PR95 review round MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 8 of the PR95 increment's review (at 082962e2) confirmed 16 findings, one high. Round 7's glued-credential chain could start a narrower family inside a wider credential or swallow a key name, and the round found four older gaps. All material findings are fixed; the password-position finding is documented as credential equality. Core projection (packages/core/src/tools/bounding.ts): - The round-7 chain and its glued-span exemption are removed; a glued credential is read at the earlier mask's boundary one pass later, as before, and a run that never settles is redacted and refused (PR95R8-SCRUB-1, TST-R8-3). - Structured summaries mask what JSON escaping hid (PR95R8-PROP-1, high): a non-string result is summarized from its JSON once every member under a credential-named key and every argument after a credential flag is masked whole, and the text rules read that JSON as one text, a value one JSON string down included, so a PEM block or a quoted value that runs across members keeps its context. A summary masks a quoted value that stops at a line break through a later quote that could close it. - The key rule reads one JSON string down: an escaped quote before the separator, a value opened by an escaped quote decoded one level, also behind escaped tabs and line breaks (where an apostrophe opens a value too). Only its escaped quote closes such a value. - A quoted value closes on its line. One that does not is decided once the pass's other masks are known (PR95R8-PROP-4): when one of them takes its line break it closes as the next pass reads it and is masked whole; otherwise it is read as an unquoted run from after its quote, to its line's end, since its quote may close a string the key sits in. - A header label at a line start inside a split credential's continuation is read through a second label view (PR95R8-PROP-2). - Two member names of one object that project alike make the journal projection unsettled, so the registry refuses the tiered call. - toText makes both texts from one media-redaction walk, and the key and scheme rules return early on text that cannot hold them. - The userinfo zone stays round 7's. A password-position key's value runs to the key rule's delimiter, through a URL's host and path, so calls that differ only there share a projection, as recognized credentials do; documented (DOC-R8-1, PR95R8-SCRUB-2). - A quoted value closes at the later of two readings: with the masks before and inside it filled, and with each mask keeping the backslashes it ends with. It applies to the key rule, the journal's closing view and the summary's closer search (SCRUB-PRE6-1, a regression against 9902e0a0 since round 6). The same rule masks the tail that an opaque value's mask unescaped behind a URL userinfo, which the author's oracles found to be a regression too. - A structured summary's walk keeps a private-key marker whose partner lies outside a member it masks whole, so a block across members is still masked (SEC-R8-PEM-1); the journal and display walks keep their whole mask. - Every summary also reads a view of the same length in which each escaped \n, \r or \t is that separator, for the split-credential, standalone and scheme-label rules, so a line-initial token in a run_command or MCP result is masked (PR95-GEN-R8P6-1, a gap 9902e0a0 shares). The journal projection reads no escape. Backstop (packages/llm/src/scrub-secrets.ts): the split probe also reads the raw text when terminal normalization changed it. A JSON-escaped separator before a quoted header value and terminal sequences that take a shape's first letter are documented limits, and a path-less hyperlink target the raw probe joins with a later @ a documented fail-closed trade-off. Tests and controls: - Discriminating cases for every change, each checked against its mutation (122 mutants: 118 killed, 4 output-neutral): structured summaries across members, line breaks and credential members, unclosed values read as runs or closed through the pass's other masks and the calls they keep distinct, one JSON level down, colliding member names (also in arrays), continuation labels, two and three PEM blocks (TST-R8-1), the scrubber's TAB retry (TST-R8-2), the fill's bracket, the escaped CR/LF and the ' delimiter (TST-R8-4), the raw split probe, header tokens a JSON escape begins, credentials before a terminal sequence masked in place, the host cut at a fragment, a real, unmocked non-settling refusal (PR95R8-PROP-3), a quoted value's two readings, the private-key markers a structured summary keeps and the escaped separators every summary reads. Core's eight-MiB cases take a case deadline instead of wall-clock bounds (CRIT-R8-2). - Twelve controls added, two removed with their mechanisms, six re-anchored and two witnesses added: 130 controls / 513 targets and witnesses in two shards of 65. Both shards ran locally with the export enabled: 1,177 files and 49.83 / 49.76 MiB. Before this commit the reviewers' generative harness, exhaustive identity and summary oracles, a distinctness oracle, the sixth pre-review's own generators, a mutation run and the affected causal controls ran against the uncommitted fix, and six internal multi-lens pre-reviews reviewed it as unreferenced commits; their findings are resolved here, and two low edge cases that 9902e0a0 shares are recorded in deferred-tasks.md. Mechanisms an earlier candidate added beyond the findings (a header escape guard, a terminal final-byte view, masking unclosed values to the line end, a trimmed opaque mask, a per-string structured summary) were withdrawn when a pre-review showed they leaked, merged identities or lost context. Docs: built-in-tools.md, effect-journal.md §11 (including the refusal's slot behaviour, CRIT-R8-1), tool-registry.md, error-handling.md and the scrubber's comments (PR95R8-PROP-5), commands.md's date (DOC-R8-2), the secrets controls README, ci.yml, the phase register's round table, ruling and gates, deferred-tasks.md's masker section, and the status lines. The maintainer's ruling of 2026-10-21 narrows this increment's acceptance bar: a round with no high or medium finding and no regression against 9902e0a0 accepts it, and each low finding is recorded in deferred-tasks.md with its file, an example and where to re-check it. The phase register's PR95 checkpoint records the ruling; every other increment keeps two clean complete cumulative rounds. Refs: ADR-0113 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 +- AGENTS.md | 5 +- CLAUDE.md | 5 +- docs/reference/cli/commands.md | 2 +- docs/reference/shared-core/built-in-tools.md | 169 ++- docs/reference/shared-core/effect-journal.md | 69 +- docs/reference/shared-core/tool-registry.md | 5 +- docs/roadmap/current.md | 9 +- docs/roadmap/deferred-tasks.md | 30 + .../phase-2.5.5-hardening-and-remediation.md | 56 +- docs/standards/error-handling.md | 128 +- packages/core/src/tools/bounding.test.ts | 1172 ++++++++++++++--- packages/core/src/tools/bounding.ts | 603 +++++++-- packages/core/src/tools/registry.test.ts | 93 +- packages/core/src/tools/registry.ts | 9 +- packages/llm/src/scrub-secrets.test.ts | 80 ++ packages/llm/src/scrub-secrets.ts | 25 +- tools/causal-controls/provenance.test.mjs | 29 +- tools/secrets-controls/README.md | 120 +- tools/secrets-controls/mutations.mjs | 167 ++- 20 files changed, 2272 insertions(+), 506 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b8bc4d5..7005f1d1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -281,7 +281,7 @@ jobs: secrets-mutations: name: Wave 2 secrets causal controls (shard ${{ matrix.shard }} of 2) # The historical 35-control, three-phase capture exceeded the former aggregate budget. - # Wave 3's PR95 increment grows the cohort to 120 controls (2026-10-20): each scrubber + # Wave 3's PR95 increment grows the cohort to 130 controls (2026-10-21): each scrubber # control runs its eight-MiB-heavy file in three phases (about 24 s locally). One export # of the whole cohort no longer fits the unchanged 64-MiB retained-evidence bound, so two # legs each run and export a disjoint round-robin half (`--shard k/2`) under the diff --git a/AGENTS.md b/AGENTS.md index a8697520..26136003 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -126,8 +126,9 @@ gate (red since at least PR #90) was cleared on 2026-10-19. A pre-implementation and the maintainer's [2026-10-19 rulings](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) give every open PR #95 finding an owner and record 29 increments (no new ADR). The PR95 increment (the scrubber residuals, including both security residuals) is implemented on `development` and under -review: rounds 1–7 were NONCLEAN and are corrected ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)), and acceptance -needs two consecutive clean rounds. Then Step 9a.1 (release map/custody substrate, #46/ADR-0111). `development` is pushed after each accepted increment. No provider, +review: rounds 1–8 were NONCLEAN and are corrected ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)). Under the maintainer's +2026-10-21 masker acceptance bar, round 9 accepts it with no high or medium finding and no regression against +base; its low findings go to deferred-tasks.md. Then Step 9a.1 (release map/custody substrate, #46/ADR-0111). `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. See [docs/roadmap/current.md](docs/roadmap/current.md) for live status. diff --git a/CLAUDE.md b/CLAUDE.md index 3885fc5d..9d4faddc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -215,8 +215,9 @@ gate (red since at least PR #90) was cleared on 2026-10-19. A pre-implementation and the maintainer's [2026-10-19 rulings](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) give every open PR #95 finding an owner and record 29 increments (no new ADR). The PR95 increment (the scrubber residuals, including both security residuals) is implemented on `development` and under -review: rounds 1–7 were NONCLEAN and are corrected ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)), and acceptance -needs two consecutive clean rounds. Then Step 9a.1 (release map/custody substrate, #46/ADR-0111). `development` is pushed after each accepted increment. No provider, +review: rounds 1–8 were NONCLEAN and are corrected ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)). Under the maintainer's +2026-10-21 masker acceptance bar, round 9 accepts it with no high or medium finding and no regression against +base; its low findings go to deferred-tasks.md. Then Step 9a.1 (release map/custody substrate, #46/ADR-0111). `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. For live status, per-PR history, milestone dates, and open obligations, see the canonical diff --git a/docs/reference/cli/commands.md b/docs/reference/cli/commands.md index 16e8a757..d392768c 100644 --- a/docs/reference/cli/commands.md +++ b/docs/reference/cli/commands.md @@ -1,6 +1,6 @@ # CLI Command Reference (`relavium`) -> Last updated: 2026-10-18 +> Last updated: 2026-10-20 - **Status**: Reference (partial — surface defined, exact flags to be finalized as the CLI is built) - **Surface**: CLI (`relavium`) diff --git a/docs/reference/shared-core/built-in-tools.md b/docs/reference/shared-core/built-in-tools.md index fa50fdc6..c49a4261 100644 --- a/docs/reference/shared-core/built-in-tools.md +++ b/docs/reference/shared-core/built-in-tools.md @@ -93,67 +93,136 @@ This is conservative pattern masking, not proof that arbitrary sensitive content absent; raw dispatch/model results, full spills and historical rows retain their existing content-sensitive handling. -A raw `?` or `#` before a URL authority's `@` begins public query or fragment text only when -the text before it is host-shaped (a dotted name or IPv4, `localhost` or a bracketed IP literal, -with an optional port) **and** an `=` assignment follows it; otherwise the userinfo is masked. -This is the same predicate as the +A raw `?` or `#` before a URL authority's `@` begins public query or fragment text only when the +text before it is host-shaped (a dotted name or IPv4, `localhost` or a bracketed IP literal, with an +optional port) **and** an `=` assignment follows it; otherwise the userinfo is masked. This is the +same predicate as the [LLM diagnostic backstop](../../standards/error-handling.md#diagnostic-secret-backstop), including its 260-character bound on the host-shaped prefix. Masked userinfo that holds such a raw `?` or `#` leaves the authority ambiguous: the userinfo may also be a query that another rule reads (`https://gateway?password=P@ss.word1`). The backstop masks through the authority's end unless a -host-shaped name follows its last `@`. This projection, like the backstop, finds every rule's -ranges on the same text and merges overlapping ranges before it replaces anything, so no rule's -mask removes the key name, scheme label, PEM marker or URL scheme another rule reads. Its -key/value and opaque-query rules read every key and parameter in a userinfo but its username's and -mask each value to its own end, through the last `@` when the value holds one -(`https://gateway?password=P@ss.word1` → `https://[redacted]`; +host-shaped name follows its last `@`. This projection, like the backstop, finds every rule's ranges +on the same text and merges overlapping ranges before it replaces anything, so no rule's mask +removes the key name, scheme label, PEM marker or URL scheme another rule reads. Its key/value and +opaque-query rules read every key and parameter in a userinfo but its username's and mask each value +they recognize — a secret key's, unquoted from six characters, or an opaque one of 24 or more +characters — to its own end: through the last `@`, and on through the host, path and query when no +delimiter of theirs comes first (`https://gateway?password=P@ss.word1` → `https://[redacted]`; `jdbc:sqlserver://db.example:1433;user=sa;password=P@ssw0rd2024;encrypt=true` → -`jdbc:sqlserver://[redacted];encrypt=true`). The username is the text before the userinfo's first -`:` when no raw `?` or `#` comes before that colon; a userinfo with no such colon has none, and a -username never reaches past an LF or TAB. A key in the username names the user, so its value does -not run on through the public host and path +`jdbc:sqlserver://[redacted];encrypt=true`; `https://user:password=@host.example/v1/a` → +`https://[redacted]`). Such a value is a recognized credential, so calls that differ only in the +host or path it runs through share a journal projection. A value no rule recognizes is masked only +as far as the userinfo's last `@`, so its text after that `@` reads as the host +(`https://svc?a=1&code=Adm1n@2024prod` → `https://[redacted]@2024prod`), where the backstop masks an +authority with no host-shaped name after its last `@` to its end. The username is the text before +the userinfo's first `:` when no raw `?` or `#` comes before that colon; a userinfo with no such +colon has none, and a username never reaches past an LF or TAB. A key in the username names the +user, so its value does not run on through the public host and path (`https://x-access-token:@github.com/o/r` → `https://[redacted]@github.com/o/r`), and a key on -the line after a split URL masks its value to its own end (`see https://\nDB_PASSWORD=P@ssw0rd123 now` → -`see https://\n[redacted] now`). The projection +the line after a split URL masks its value to its own end +(`see https://\nDB_PASSWORD=P@ssw0rd123 now` → `see https://\n[redacted] now`). The projection otherwise differs from that backstop by design: its authority also ends at `"`, `'`, `,`, `}` and -`]`, so userinfo that contains one of them — an already-masked `[REDACTED]` password among them — -is left as it is; it recognizes any scheme of up to 32 characters rather than a finite list; and a +`]`, so userinfo that contains one of them — an already-masked `[REDACTED]` password among them — is +left as it is; it recognizes any scheme of up to 32 characters rather than a finite list; and a userinfo with no password is kept as a vanity handle below 16 characters and masked from 16 on, under every scheme. -LF/TAB inside a recognized token-shaped credential extends the masked original span. -An ambiguous token-alphabet continuation after LF/TAB is conservatively masked with -that credential. An LF or TAB right after a URL's `://`, right before its userinfo's `@` or inside -its `scheme://` belongs to the split userinfo, which is masked as one span. A credential glued to -the end of a fixed-length one (`AKIA…ASIA…`, `AKIA…ghp_…`) is a credential of its own, so a run of -them is masked whole in one pass. A variable-length token is masked to the end of its alphabet, so -a second token glued to it with no separator, whose prefix the first body takes only in part -(`ghp_ghp_`), is masked only through that part, and the rest of the second token stays -visible. A key's value runs -through every span an earlier rule masks, line breaks included, as through an applied mask, so a -quoted value that carries a split credential or a PEM block is masked whole. A key inside another -key's value, and a scheme label inside another label's token, start a credential of their own -(`token=a>password: ` → `[redacted]`; `Bearer a/Bearer ` → `Bearer [redacted] [redacted]`), -and an LF/TAB-split header label or a scheme label may start right after a credential or a PEM -block, as after an applied mask. A scheme label -that takes a URL scheme as its token no longer hides that URL's password -(`token postgres://app:pw@db` → `token [redacted]://[redacted]@db`). The projection repeats until -its output stops changing, at most three passes, and becomes `[redacted]` if it has not settled, so -projecting a projection changes nothing. A tiered call whose journal projection has not settled is -refused before it is prepared ([tool registry](tool-registry.md#resolution--the-dispatch-lifecycle)), so that constant -never stands for two different arguments in the journal. Every rule's scan stays linear in the -admitted text, and none uses a backtracking pattern whose stack an admitted value can exhaust: a -quoted value is read by a character scan to its closing quote. -Ordinary spaces and punctuation separate public prose; the masker does not discard the whole -diagnostic because a credential appears. Supported split -shapes are recognized **before** complete-text one-line normalization, by the projection's full -fixed point, followed by -ordinary recognition and length clipping. This preserves multiline quoted-value -recognition without exposing a split token's suffix. Public argument bytes are not +LF/TAB inside a recognized token-shaped credential extends the masked original span. An ambiguous +token-alphabet continuation after LF/TAB is conservatively masked with that credential; an +LF/TAB-split header label that begins a line inside such a continuation is still read (`sk-…` ending +a line before a split `api-k\tey:` label). A credential that begins such a line is masked as far as +the continuation reaches, so when the continuation takes the start of its prefix and the credential +is itself split again, its rest stays visible (`hf_…` ending a line before an LF-split `glpat-…`). +An LF or TAB right after a URL's `://`, right before its userinfo's `@` or inside its `scheme://` +belongs to the split userinfo, which is masked as one span. A credential glued to the end of another +starts at the earlier mask's boundary, so it is read on the next pass; a run of three or more glued +credentials (`AKIA…ASIA…AKIA…`) therefore does not settle (below). When the earlier mask takes the +start of the glued credential's prefix — for example a variable-length token masked to the end of +its alphabet (`ghp_ghp_`) or its LF/TAB continuation, a scheme label's token, or a key's +value masked to its delimiter — the rest of that credential stays visible, as it did before this +projection's corrections; so does the first line of a glued credential split by an LF or TAB whose +continuation another rule masks in the same pass, which leaves that line below its family's minimum +(`AKIA…ghp_<15 characters>\ntoken=…`). A quoted value closes on its line, at the later +of two readings: one with the masks before and inside it filled, as the next pass reads them, and one +where each mask keeps the backslashes it ends with, as the text has them. So a mask that takes the +backslash escaping a later quote neither closes the value early +(`password="Bearer\nabcdefgh token=xyz\"tail" more` → `[redacted] more`) nor, when that quote is the +value's only closer, leaves it open (`password="say \"…?sig=<24 or more characters>\"` → +`[redacted]`), in a summary too. One that does not close on its line is decided once the pass's other +masks are known: when one of them takes its line break (a scheme label's token, a nested key's value), +it closes as the next pass reads it and is masked whole, as before this rule; otherwise it is read as an unquoted value from after its quote — a run +of at least six characters up to whitespace, a quote, `,`, `;` or `&` — and no further than its +line's end, since its quote may instead close a string the key sits in: the text after such a quote +stays (`stop logging "token=" values`), so calls that differ only there keep distinct projections, +and the later words or lines of a value that never closes or closes on a later line stay visible +(`password="firstline\nsecond"` → `[redacted]\nsecond"`; a first line under six characters stays as +well). A run is masked from its key in the pass that finds it, so a key or a line break that only a +later pass reads, behind a glued credential's mask +(`password="AKIA<16>abcdefghijklm_token: \n…"`), goes with the run and the value behind it stays. +A value opened by an escaped quote (`\"…\"`, as JSON inside a JSON string writes it) is read one +decoding level down, also behind escaped tabs and line breaks after its separator +(`password:\t\"…\"`), where an apostrophe, which JSON leaves unescaped, opens a value too +(`password:\n '…'`); a raw double quote behind them, as `util.inspect` or `repr` print one +(`password:\t"…"`), does not, a documented limit. A key inside an opaque query value loses its value +to that value's mask, which runs to the escaped quote and takes its backslash +(`?state=<24 or more characters>;password=\"…\"` keeps the quoted text), a documented limit. Only +its escaped quote closes it: a raw line break, behind a backslash or not, or a raw quote of its own +kind ends the string around it, so a value that has not closed by then is read as a run, like one +that does not close on its line (`"stop logging \"token=\" values"` stays as it is); a raw quote of +the other kind does not end that string, since an apostrophe is ordinary text in a JSON string, so +inside a string delimited by that kind the value runs on. One quoted two or more JSON levels down is +outside the shapes, and so is a single-quoted value inside a JSON string whose apostrophe is escaped +(`'ab\'…'` written as `'ab\\'…'`): JSON doubles the backslash, so the value closes at that +apostrophe and the rest stays. A key's value runs through every span an earlier rule masks, line +breaks included, as through an applied mask, so a quoted value that carries a split credential or a +PEM block is masked whole. A key word must start within the first 33 characters of its word, counted +from the last word boundary (a `-` starts one, a `_` does not), so a longer name is no key +(`MY_APPLICATION_PRODUCTION_DATABASE_PASSWORD=…` stays). A key's mask starts where its word starts, +so a credential glued to the front of a key name (`password=…`) is masked with that key's +value and stays when the value is empty or too short to mask. A key inside another key's value, and +a scheme label inside another label's token, start a credential of their own +(`token=a>password: ` → `[redacted]`; `Bearer a/Bearer ` → +`Bearer [redacted] [redacted]`), and an LF/TAB-split header label or a scheme label may start right +after a credential or a PEM block, as after an applied mask. Core's scheme rule reads one unquoted +label followed by whitespace, so a self-quoted label (`"Bearer" `) or a concatenated run +(`BearerBearer `), which the backstop masks, is outside its shapes. A scheme label that takes a +URL scheme as its token no longer hides that URL's password (`token postgres://app:pw@db` → +`token [redacted]://[redacted]@db`). The projection repeats until its output stops changing, at most +three passes, and becomes `[redacted]` if it has not settled — as a run of glued credentials or a +key whose short value names the next key can need more — so projecting a projection changes nothing; +a display field that does not settle (three glued fixed-length ids suffice) therefore shows as +`[redacted]` whole, and of such calls only a tiered one is refused. A tiered call whose journal +projection has not settled, or whose arguments hold two member names of one object that project +alike, which leaves one of the members, is refused before it is prepared +([tool registry](tool-registry.md#resolution--the-dispatch-lifecycle)), so neither that constant nor +a lost member stands for two different arguments in the journal. Every rule's scan stays linear in +the admitted text, and none uses a backtracking pattern whose stack an admitted value can exhaust: a +quoted value is read by a character scan no further than its line, and a summary decides its values +that stop at a line break by one backward scan. Ordinary spaces and punctuation separate public +prose; the masker does not discard the whole diagnostic because a credential appears. A summary uses +the projection with one difference, since a summary is shown, never compared: a quoted value that +stops at a line break is masked through the nearest later quote that could close it outside the +values the rule masks or, when every such quote is inside one, through the last of them, and only +else to the end of its line, in whichever of the two readings reaches further; one that reaches the end of the string around it or of the text without +closing is masked to that end. A structured (non-string) result is summarized from its JSON once +every member under a credential-named key and every argument after a credential flag is masked whole +(a count such as `input_tokens` too, the text rules then masking the member's name with it); such +a mask keeps a private-key BEGIN or END marker whose partner lies outside the member, so a block that +runs on into or out of it is masked across members. The text rules read that JSON as one text, a +value one JSON string down included, so a JSON-escaped line break is no delimiter and a PEM block or +a quoted value that runs across members keeps its key and markers; a member masked whole is not read +for a quoted value that runs on from it. Every summary also reads its text a second time with each +escaped line feed, carriage return or tab (`\n`, `\r`, `\t`, behind any run of backslashes) as that +separator, for the rules a separator decides: a credential or a scheme label right after one, and a +credential split by one, are masked as in raw text (`{"stdout":"keys:\nsk-…"}` → +`{"stdout":"keys:\n[redacted]"}`), while a header label split by one is not read; the journal +projection reads no escape. Supported +split shapes are recognized **before** complete-text one-line normalization, by that projection's +full fixed point, followed by ordinary recognition and length clipping. This preserves multiline +quoted-value recognition without exposing a split token's suffix. Public argument bytes are not globally whitespace-normalized for journal identity. The -[journal contract](effect-journal.md#11-secrets-what-a-row-may-hold) owns recognized- -credential equality and persisted compatibility; diagnostic masking grants no -permission to repeat an effect. +[journal contract](effect-journal.md#11-secrets-what-a-row-may-hold) owns recognized-credential +equality and persisted compatibility; diagnostic masking grants no permission to repeat an effect. ## Subprocess environment diff --git a/docs/reference/shared-core/effect-journal.md b/docs/reference/shared-core/effect-journal.md index 46b2ef7e..7ab75130 100644 --- a/docs/reference/shared-core/effect-journal.md +++ b/docs/reference/shared-core/effect-journal.md @@ -369,34 +369,49 @@ no fallback to the old digest or automatic retry permission. Secret and inline-media redaction remain deliberate projection equality; raw credential bytes are never added to the digest to distinguish calls. -Recognized URL userinfo is masked through the authority's last `@`, preserving the host, -path, query and fragment; the values of the keys and query parameters in it, its username's -excepted, are masked to their own ends as well, through that `@` when a value holds one. -Recognized opaque URL-query values are masked through their delimiter, including values -longer than 512 characters. Quoted secret-key values honor escaped quotes and backslashes -through the actual closing quote. A recognized value must not leave a credential suffix -in the projection. This remains a finite shape detector, not a claim to identify arbitrary -unlabelled secrets. Existing old digests are not retroactively erased or rewritten. - -A raw `?` or `#` before the authority's `@` is public only after a host-shaped prefix that an -`=` assignment follows ([built-in tools](built-in-tools.md#finite-diagnostic-credential-masking)). -Wave 3's PR95 increment tightened that predicate without a new digest domain. It also merges -the ranges of every rule before replacing anything, so a userinfo is masked together with the -values of every key and query parameter in it but its username, which never reaches past an LF or -TAB, and no mask removes another credential's context; reads a key or scheme label inside another -value as a credential of its own, and a value through earlier masks; masks userinfo split by an LF -or TAB at either edge or inside its `scheme://`; masks a credential glued to the end of a -fixed-length one as a credential of its own; reads a split header label or a scheme label right -after a credential or a PEM block; and repeats the projection until it is stable. -These change a `v5:` projection only for the arguments they project differently, such as -userinfo previously treated as public or a credential an earlier mask had hidden: a resumed run whose -committed effect carried one now computes a different digest and refuses matching prepare -instead of replaying, which is the fail-closed outcome of any digest difference. Every other -`v5:` row keeps its digest. A projection that has not settled after three passes becomes the -constant `[redacted]` in diagnostics, but it never becomes a journal identity: the registry -refuses such a tiered call with `ToolArgsInvalidError` before its prepare +Recognized URL userinfo is masked through the authority's last `@`, preserving the host, path, query +and fragment; the values the key and opaque-query rules recognize in it, its username's excepted, +are masked to their own ends as well, through that `@` and on through the host, path and query when +no delimiter of theirs comes first, while a value no rule recognizes ends at that `@` and the text +after it stays as the host +([built-in tools](built-in-tools.md#finite-diagnostic-credential-masking)). Recognized opaque +URL-query values are masked through their delimiter, including values longer than 512 characters. +Quoted secret-key values honor escaped quotes and backslashes through the actual closing quote on +their line. A recognized value must not leave a credential suffix in the projection; what a rule +does not recognize whole, such as the later words of a quoted value that does not close on its line, +is listed with the shapes +([built-in tools](built-in-tools.md#finite-diagnostic-credential-masking)). This remains a finite +shape detector, not a claim to identify arbitrary unlabelled secrets. Existing old digests are not +retroactively erased or rewritten. + +A raw `?` or `#` before the authority's `@` is public only after a host-shaped prefix that an `=` +assignment follows ([built-in tools](built-in-tools.md#finite-diagnostic-credential-masking)). Wave +3's PR95 increment tightened that predicate without a new digest domain. It also merges the ranges +of every rule before replacing anything, so a userinfo is masked together with the recognized values +of the keys and query parameters in it but its username, which never reaches past an LF or TAB, and +no mask removes another credential's context; reads a key or scheme label inside another value as a +credential of its own, and a value through earlier masks; masks userinfo split by an LF or TAB at +either edge or inside its `scheme://`; reads a quoted value one JSON string down too, and one that +does not close on its line, or before the end of its string, as an unquoted run that stops there — +unless another mask of the pass takes its line break and it closes, as the next pass read it before +(at the later of its readings with and without the masks' trailing backslashes) — so the text after a quote that may close a string around the key keeps calls distinct; reads a +split header label or a scheme label right after a credential or a PEM block, and a header label at +a line start inside a split continuation; and repeats the projection until it is stable. These +change a `v5:` projection only for the arguments they project differently, such as userinfo +previously treated as public or a credential an earlier mask had hidden: a resumed run whose +committed effect carried one now computes a different digest and refuses matching prepare instead of +replaying, which is the fail-closed outcome of any digest difference. Every other `v5:` row keeps +its digest. Text a rule reads as part of a credential's value — such as the host, path and query a +password-position key's value runs through — is that credential's, so calls that differ only there +share a projection, as recognized credentials always do. A projection that has not settled after +three passes becomes the constant `[redacted]` in diagnostics, but it never becomes a journal +identity: the registry refuses such a tiered call with `ToolArgsInvalidError` before its prepare ([tool registry](tool-registry.md#resolution--the-dispatch-lifecycle)), so two different calls -cannot share it and replay one another's results. +cannot share it and replay one another's results. Two member names of one object that project alike +would leave one member for two argument sets, so such a projection counts as not settled and is +refused the same way. Like any refusal before the prepare, it does not consult a row an earlier +version committed at that slot; the corrected call takes the next slot, as +[§14](#14-known-limitations)'s slot instability already allows ([§4](#4-the-resume-gate)). A recognized structural credential key masks its entire value before traversal. A partial inner text match does not make the remaining value public. Only an anchored Bearer, Basic diff --git a/docs/reference/shared-core/tool-registry.md b/docs/reference/shared-core/tool-registry.md index bf2febe5..97bf1e13 100644 --- a/docs/reference/shared-core/tool-registry.md +++ b/docs/reference/shared-core/tool-registry.md @@ -258,7 +258,8 @@ host is touched once, in the middle. The durable [effect bracket](effect-journal.md) prepares a tiered call after approval and settles it after model-facing bounding. Its prepare carries the journal projection of the effective arguments; when that projection has not settled within three passes (credential-shaped text with no stable -redacted form, [built-in-tools.md](built-in-tools.md#finite-diagnostic-credential-masking)), the call +redacted form, [built-in-tools.md](built-in-tools.md#finite-diagnostic-credential-masking)), or when two +member names of one object project alike and would leave one member for two argument sets, the call is refused with a loop-correctable `ToolArgsInvalidError` before anything is prepared or dispatched. It names the affected arguments through the same masker, since an open schema lets the model choose an argument's name. Immediately before step 5, after any asynchronous approval and preparation, @@ -485,7 +486,7 @@ codes by [sse-event-schema.md](../contracts/sse-event-schema.md#error-code-taxon | `UnknownToolError` | id not an exact match | `tool_failed` | fatal (loop-correctable first) | | `ToolPolicyError` | a guardrail / grant denial — `not_granted`, `provider_executed`, `command_not_allowed`, `domain_not_allowed`, `insecure_url`, `gate_required`, `media_scope_denied` (the full `ToolPolicyDenyReason` union; `media_scope_denied` is `read_media`'s scope-set denial, [ADR-0044](../../decisions/0044-media-access-governance-read-media-save-to-cost.md) §1) | `tool_denied` | **fatal** — EXCEPT `media_scope_denied` is `recoverable` (a Step-14 SCOPE denial; see the note below) | | `ToolDeniedByUserError` | an interactive **per-tool approval** denial ([ADR-0057](../../decisions/0057-cli-chat-modes-and-per-tool-approval.md) EA3) — `user_rejected` (rejected by the user / mode policy), `no_approval_hook` (fail-closed: a governed dispatch under an active regime with no confirm hook wired), `approval_error` (fail-closed: the hook threw a non-abort error, so consent could not be obtained) | `tool_denied` | **fatal** (never retried; not loop-correctable — re-asking re-prompts/re-denies, like `ToolPolicyError`) | -| `ToolArgsInvalidError` | effective args fail `parseArgs` / secret-taint, or a tiered call's journal projection does not settle | `validation` | fatal (loop-correctable first) | +| `ToolArgsInvalidError` | effective args fail `parseArgs` / secret-taint, or a tiered call's journal projection does not settle or projects two member names alike | `validation` | fatal (loop-correctable first) | | `ToolUnavailableError` | the required `ToolHost` capability is absent (host/config gap, not the model's fault) | `tool_unavailable` | **fatal** (names the tool + the unwired arm actionably — never a bare `internal`; EA1, [ADR-0055](../../decisions/0055-cli-host-capability-seam-tool-environment-factory.md)) | | `ToolExecutionError` | the host capability threw a non-cancel error (cause kept off the message, for logs) | `tool_failed` | retryable (node budget) | | *(AbortSignal abort)* | the run was cancelled mid-tool | `cancelled` | fatal (cancel path, not `tool_failed`) | diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index 4c3cd756..1c872fc3 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -1,7 +1,7 @@ # Current state > Status: Living -> Last updated: 2026-10-20 +> Last updated: 2026-10-21 - **Related**: [README.md](README.md), [phases/phase-2.5-cli-consolidation.md](phases/phase-2.5-cli-consolidation.md), [phases/phase-2.5.5-hardening-and-remediation.md](phases/phase-2.5.5-hardening-and-remediation.md), [phases/phase-2-cli.md](phases/phase-2-cli.md), [deferred-tasks.md](deferred-tasks.md), [../project-structure.md](../project-structure.md), [../tech-stack.md](../tech-stack.md) @@ -165,9 +165,10 @@ Wave 3 implementation continues on `development`: Steps 1–8 are merged. A pre- of Steps 9–19 and the maintainer's rulings are recorded in the [2026-10-19 rulings](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) (no new ADR; 29 recorded increments). **The PR95 increment is implemented on `development` and under review** — the PR #95 -scrubber residuals, including the two security residuals; rounds 1–7 were NONCLEAN and are corrected, as its -[implementation checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20) records, and acceptance needs two consecutive clean rounds after the last -material fix. **Step 9a.1** follows, the first increment of the release map/custody substrate (#46, ADR-0111), whose +scrubber residuals, including the two security residuals; rounds 1–8 were NONCLEAN and are corrected, as its +[implementation checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20) records. Under the maintainer's 2026-10-21 masker acceptance bar, round 9 accepts it with no high or medium +finding and no regression against base, its low findings recorded in +[deferred-tasks.md](deferred-tasks.md#pr-95-masker-low-edge-cases--approved-deferrals-2026-10-21). **Step 9a.1** follows, the first increment of the release map/custody substrate (#46, ADR-0111), whose 9a.1–9d increments carry a dedicated archive/crypto security sitting. Steps 10–19 follow the [increment table](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) and end with whole-wave acceptance at Step 19. `development` is pushed after each accepted increment; PR #95 no longer bounds the diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index bc2a7f07..281e471f 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -448,6 +448,36 @@ measured or counted here. can return a raw signal code above 9 that the physical exit never uses, a note for future embedders. No action is proposed for any of these. +## PR #95 masker low edge cases — approved deferrals (2026-10-21) + +Under the [masker acceptance bar](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20) +that the maintainer set on 2026-10-21, a low finding in the open-ended masker work +(`packages/llm/src/scrub-secrets.ts` and core's twin projection in `packages/core/src/tools/bounding.ts`) is +recorded here instead of being corrected in the review loop. None of these entries is a regression against base +`9902e0a0`, and none was judged high or medium. The shapes that neither masker reads by design stay described in +their canonical homes, [built-in-tools.md](../reference/shared-core/built-in-tools.md) for core and +[error-handling.md](../standards/error-handling.md) for the scrubber; this list holds the review findings and the +re-checks they need. **Trigger for every entry:** a real tool result, journal entry or diagnostic that carries the +shape, or the next change to the rule the entry names. The generative harnesses that found them are not part of +the repository, so each entry names the input and the property to re-check by hand or in a test. + +- [ ] **Approved deferral (maintainer, 2026-10-21). A header label split by a JSON escape is not joined in a + summary.** *Low, unchanged from base.* *(author)* `boundForModel({ stdout: 'api-k\tey: ' })` shows the + value in its summary, while the same text as a string result is `[redacted]`. The summary's second reading + (`visitEscapedSeparators`) runs the standalone, scheme-label and split-credential rules, not the split + header-label join, which rewrites the text instead of reporting ranges. **Re-check:** whether a real tool output + splits a header label across a line break; a fix reports the joined labels as ranges of the escaped view. + *(low · packages/core/src/tools/bounding.ts)* +- [ ] **Approved deferral (maintainer, 2026-10-21). A private-key block split across array elements shows its + body in the journal and display projections.** *Low, unchanged from base.* *(author)* + `redactSecretShapedValue(['--private-key', '', 'MIIE…', ''])` keeps `'MIIE…'` and the + END marker, and `{ argv: ['ssh', '-i', '\nMIIE…', '…\n'] }` stays whole. Both + projections read each string alone; only a summary reads a structured value's JSON as one text. **Re-check:** + `redactSecretShapedWalk` under `projectSecretShapedValue` and `redactSecretShapedValue`; a fix changes `v5:` + journal digests, so it needs the compatibility review of + [effect-journal.md](../reference/shared-core/effect-journal.md#11-secrets-what-a-row-may-hold) §11. + *(low · packages/core/src/tools/bounding.ts)* + ## Phase 2.6.5 deferrals — `CR-93` and `CR-95` (long half), recorded 2026-09-06; `CR-70`, `CR-94` (remainder) and the in-turn overflow continuation, recorded 2026-09-14; the errored-or-aborted turn residual, the pre-upgrade free-page bytes and the estimator's calibration, recorded 2026-09-18 **Phase closure — 2026-10-11:** W8 merged to `main` in PR #92 and Phase 2.6.5 is complete diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 35f6d487..82640a11 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -9424,8 +9424,8 @@ carries the same owners. the CLI's authored-content detector, the secrets causal controls and the canonical documents. Base `9902e0a0`; commits `38e10d42` (Docs-0, the rulings), `dda3b45e` (implementation) and the corrections `ef3ded0d`, `bdd292a6`, `6fb7e9d9`, `b2c50e88`, `6b2e94dd` (round 5, which added this -section), `b89ac62e` (round 6) and the round-7 commit. The commits stay local on `development` until -the increment is accepted. +section), `b89ac62e` (round 6), `082962e2` (round 7) and the round-8 commit. The commits stay local on +`development` until the increment is accepted. **Review rounds.** Each round is a NEW independent complete cumulative review of `9902e0a0..HEAD`, every reviewer in its own worktree detached at the reviewed head: scrubber correctness, tests and causal @@ -9442,20 +9442,42 @@ stay local and Git-ignored. | 5 | `b2c50e88` | NONCLEAN | 20 (2 high) | core's round-4 authority extent erased the context its sequential rules read, so core now masks an ambiguous userinfo with exactly the query values its later rules mask whole; its split view masks URL userinfo before scheme labels; the LLM split scan retries a kept candidate glued across a line break; tested fixed-point helpers; untested pattern pieces gained discriminating tests | | 6 | `6b2e94dd` | NONCLEAN | 21 (9 high) | round 5's value mirror was quadratic and still erased context, and its split view dropped the value mask; core now merges every rule's ranges on one text, as the scrubber does, reads a key or scheme label inside another value and a value through earlier masks, reads a split header label or a scheme label right after a credential, and scans PEM blocks linearly; the scrubber's nine-label bound also reads the raw text and its literal query names follow `#`; both fail-closed values are tested | | 7 | `b89ac62e` | NONCLEAN | 17 (2 high) | only a userinfo's username owns its keys, never past an LF or TAB, so a key in its password or parameter position or on the next line masks its value through the last `@`; quoted values are found by a character scan that an admitted 8-MiB value cannot exhaust; a credential glued to a fixed-length one is masked in the same pass; summaries use the projection's fixed point; a tiered call whose journal projection does not settle is refused before its prepare; the secrets controls run as two CI shards so that each evidence export fits the unchanged 64-MiB bound, which the single export had exceeded; untested pieces gained discriminating tests | - -**Author gates and controls.** Every corrected head passed `pnpm run ci` and the affected suites under -`CI=true GITHUB_ACTIONS=true`, plus local differential, exhaustive and admitted-size harnesses that -found no new leak against the previous head. Until round 7 no author gate ran the evidence export, -which only the dedicated CI job enables, so the 110-control family's single export, about 77 MiB -against the unchanged 64-MiB bound, would have failed that job. From round 7 the family is 120 -controls / 360 phases / 501 targets and witnesses, run as two 60-control shards that each project -1,087 of the unchanged 2,000 retained files; both shards ran locally with +| 8 | `082962e2` | NONCLEAN (integration and security clean) | 16 (1 high) | round 7's glued-credential chain cut wider credentials and swallowed key names, and is gone; a structured result's summary reads its JSON as one text once credential members are masked; quoted values are read one JSON string down, and one that does not close on its line is read as an unquoted run from after its quote, no further than that line's end (one JSON string down, its string's end), unless another mask of the pass takes its line break and it closes; a summary masks one stopped by a line break through a later quote instead; a header label at a line start inside a split continuation is read; the scrubber's split probe also reads the raw text; a password-position key's extent is documented as credential equality. Before the commit the author ran the reviewers' method on the candidate and six internal pre-reviews: the first found three of the draft's alternatives unsound (a trusted host-shaped tail, quoted values across lines, a quadratic line-start scan), and they were dropped; the second (17 findings, none high) led to the summary's through-the-closer masking, colliding member names refused as unsettled and escaped separators and opaque values before escaped quotes; the third (27 findings, two high) found that the round's own additions failed — masking a value that never closes to its line end merged journal identities, and the scrubber's escaped header separator and terminal final-byte view let a token leak or blanked coloured diagnostics — so such a value is read as an unquoted run, the two scrubber additions are withdrawn and documented as limits, and a structured summary's walk became linear at a small constant factor; the fourth (16 findings, two medium; the critic clean, with all 128 controls run) found that the run and the opaque mask's kept backslash showed values that earlier versions masked whole in their next pass, so such a value is decided once the pass's other masks are known and the opaque mask takes the backslash again, and gaps in tests and wording; the fifth (19 findings, one high; the critic clean) found that projecting a structured result's strings one by one removed the PEM markers and keys that reading its JSON needs across members, so a structured summary reads its JSON as one text again and the summary-only runs over line breaks are withdrawn; the sixth (four findings and a duplicate, one medium; the integration lens and the critic clean) found that the closing view and the summary's closer read a quote's escape after a nested key's value had taken its backslash, and that a structured summary's walk dropped the private-key markers a block across members needs, both regressions against base, and that a structured summary read no JSON-escaped line break or tab, a gap base shares, so a quoted value closes at the later of its readings with and without the masks' trailing backslashes, which also masks the tail an opaque value's mask unescaped (behind a URL userinfo, a further regression the author's oracles found), the walk keeps a marker whose partner lies outside its member, and every summary also reads its escaped separators; two low edge cases base shares went to deferred-tasks.md under the maintainer's 2026-10-21 bar | + +**Author gates and controls.** Every corrected head passed `pnpm run ci` and the affected suites +under `CI=true GITHUB_ACTIONS=true`, plus local differential, exhaustive and admitted-size harnesses +that found no new leak against the previous head beyond the classes a round documents (in round 8, +glued credentials that round 7's chain had masked in one pass, and a key that only such a +credential's mask exposes inside a value that does not close). Until round 7 no author gate ran the +evidence export, which only the dedicated CI job enables, so the 110-control family's single export, +about 77 MiB against the unchanged 64-MiB bound, would have failed that job. From round 7 the family +is 120 controls / 360 phases / 501 targets and witnesses, run as two 60-control shards that each +project 1,087 of the unchanged 2,000 retained files; both shards ran locally with `RELAVIUM_ACCEPTANCE_EVIDENCE` set, and each export held 1,087 files and measured 45.50 MiB and -45.48 MiB, its final record included. [tools/secrets-controls/README.md](../../../tools/secrets-controls/README.md) -records each round's control changes and, from round 5, the rule that pieces inside a mechanism's -pattern are pinned by discriminating tests rather than by new controls. - -**Remaining gates.** The increment is not independently accepted: two NEW consecutive clean complete -cumulative rounds after the last material fix, with a clean separate security sitting, remain. Then -`development` is pushed and its CI observed, and the PR #95 items in +45.48 MiB, its final record included. From round 8 the family is 130 controls / 390 phases / 513 +targets and witnesses in two shards of 65 controls (1,177 files each), whose exports measured +49.83 MiB and 49.76 MiB. From round 8 the author also runs the reviewers' own +generative harness (a planted-token oracle, idempotence, JSON round trips, prose preservation, +admitted-size timing and a search for projections that do not settle) and an internal multi-lens +pre-review on every correction before its commit. +[tools/secrets-controls/README.md](../../../tools/secrets-controls/README.md) records each round's +control changes and, from round 5, the rule that pieces inside a mechanism's pattern are pinned by +discriminating tests rather than by new controls. + +**Maintainer ruling — the masker acceptance bar, 2026-10-21.** After eight external rounds and six +internal pre-reviews, the maintainer narrowed this increment's acceptance bar. A pattern-based masker +cannot be complete, and each aggressive round found another corner case while Step 9 waited. A round +is clean for this increment when it reports no high or medium finding and no regression against base +`9902e0a0` (text that base masked and the reviewed head shows). Each low finding is recorded in +[deferred-tasks.md](../deferred-tasks.md#pr-95-masker-low-edge-cases--approved-deferrals-2026-10-21) +with its file, a minimal example and where to re-check it, instead of being corrected in the loop. +The increment closes with the sixth internal pre-review and one external round (round 9) under that +bar. The ruling covers only the open-ended masker work: `packages/llm/src/scrub-secrets.ts`, core's +twin projection in `packages/core/src/tools/bounding.ts` and their controls. Every other increment +keeps the approved bar of two NEW consecutive clean complete cumulative rounds after the last +material fix. + +**Remaining gates.** The increment is not independently accepted. Round 9 remains: a NEW complete +cumulative round, with its separate security lens, that meets the 2026-10-21 bar. Then `development` +is pushed and its CI observed, and the PR #95 items in [deferred-tasks.md](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18) close. diff --git a/docs/standards/error-handling.md b/docs/standards/error-handling.md index 8ce0979a..2331e861 100644 --- a/docs/standards/error-handling.md +++ b/docs/standards/error-handling.md @@ -129,18 +129,19 @@ message, and the backstop masks a finite set of secret **shapes**, not arbitrary content. - **Shapes.** OpenAI/Anthropic `sk-…` and Google `AIza…` key prefixes; a finite list of literal - secret query names, after `?`, `&` or `#` (an OAuth implicit-flow `#access_token=`); the exact `api-key`, `x-api-key` and `x-goog-api-key` header names, also - in quoted, JSON-encoded and JSON-in-JSON (`\"x-api-key\":\"…\"`) objects; and - `Bearer`/`Basic` authentication values. A header value may begin with up to eight (possibly - quoted, escaped, punctuation-glued or LF/TAB-split) `Bearer`/`Basic` labels — a doubled - `Bearer Bearer` is a common misconfiguration. Its token may begin with any run of (escaped) - quotes and ends at a delimiter, at escapes before one, or before a nested header name, which - then masks its own value. A label may be closed by its own quote before its token - (`"Bearer" `), and a run of up to eight concatenated labels (`BearerBearer`) counts as - one label. No label takes another label as its token, even one glued behind punctuation, - quoted or concatenated, so a standalone value with any number of separately stacked labels - masks the token after the last one, and a label glued to a token behind a non-word character - masks its own token. + secret query names, after `?`, `&` or `#` (an OAuth implicit-flow `#access_token=`); the exact + `api-key`, `x-api-key` and `x-goog-api-key` header names, also in quoted, JSON-encoded and + JSON-in-JSON (`\"x-api-key\":\"…\"`) objects; and `Bearer`/`Basic` authentication values. A header + value may begin with up to eight (possibly quoted, escaped, punctuation-glued or LF/TAB-split) + `Bearer`/`Basic` labels — a doubled `Bearer Bearer` is a common misconfiguration. Its token may + begin with any run of (escaped) quotes and ends at a delimiter, at escapes before one, or before a + nested header name, which then masks its own value. A label may be closed by its own quote before + its token (`"Bearer" `), and a run of up to eight concatenated labels (`BearerBearer`) + counts as one label. No label takes as its token another label that whitespace or the end of the + text follows, even one glued behind punctuation, quoted or concatenated, so a standalone value + with any number of separately stacked labels masks the token after the last one, and a label glued + to a token behind a non-word character masks its own token; a second label that other punctuation + follows (`Bearer Bearer;`) is masked as a token, which only over-masks. - **Masking.** Every rule, the literal query names included, finds its matches in the original text, and the masked ranges of all rules are merged before anything is replaced, so no rule can erase the context another rule needs. A header, label or query rule masks only its token: @@ -153,68 +154,79 @@ content. until its output stops changing, at most three passes, and blanks the whole diagnostic if it has not settled: a second scrub changes nothing. - **URL userinfo.** The `?`/`#` rule decides where userinfo ends: a raw `?` or `#` before the - authority `@` begins public query or fragment text only when the text before it is host-shaped - (a dotted name or IPv4, `localhost` or a bracketed IP literal, with an optional port, at most - 260 characters in all) **and** an `=` assignment follows it, as in + authority `@` begins public query or fragment text only when the text before it is host-shaped (a + dotted name or IPv4, `localhost` or a bracketed IP literal, with an optional port, at most 260 + characters in all) **and** an `=` assignment follows it, as in `https://host.example?contact=a@b.example`. Masked userinfo that holds such a raw `?` or `#` - leaves the authority ambiguous: unless a host-shaped name follows its last `@`, the text there - may be the rest of a query value (`https://svc?password=P@ss`), so the mask runs through the - authority's end, the next `/`, whitespace, quote, `,` or `}`. The scheme rule then decides - what is masked: `http(s)`, `ws(s)` and `ftp(s)` mask any userinfo, while the connection-string schemes - `postgres`, `postgresql`, `mysql`, `mariadb`, `redis`, `rediss`, `mongodb`, `mongodb+srv`, - `amqp`, `amqps`, `ssh`, `sftp`, `ldap`, `ldaps`, `smtp` and `smtps` mask only colon-bearing - userinfo (`user:pass@`, `:pass@`), so a bare user such as `ssh://git@host` (or a colonless + leaves the authority ambiguous: unless a host-shaped name follows its last `@`, the text there may + be the rest of a query value (`https://svc?password=P@ss`), so the mask runs through the + authority's end, the next `/`, whitespace, quote, `,` or `}`. The scheme rule then decides what is + masked: `http(s)`, `ws(s)` and `ftp(s)` mask any userinfo, while the connection-string schemes + `postgres`, `postgresql`, `mysql`, `mariadb`, `redis`, `rediss`, `mongodb`, `mongodb+srv`, `amqp`, + `amqps`, `ssh`, `sftp`, `ldap`, `ldaps`, `smtp` and `smtps` mask only colon-bearing userinfo + (`user:pass@`, `:pass@`), so a bare user such as `ssh://git@host` (or a colonless `redis://#TOKEN@host`) stays visible. Userinfo whose password is exactly masked stays as it is: beside any username under a connection-string scheme, or beside a masked or empty username under any scheme; an empty password, or text glued after a masked one, is still masked. The authored-content detector's projection of `postgres://app:{{secrets.password}}@db` to `postgres://app:[REDACTED]@db` is therefore a fixed point. Core's journal projection applies the same `?`/`#` predicate and also merges its rules' ranges, but masks a userinfo together with the - values of the keys and query parameters in it, its username's excepted, each to its own end, - rather than through the authority's end + values its key and opaque-query rules recognize in it, its username's excepted, each to its own + end, and ends a value no rule recognizes at the last `@`, whose text after it stays as the host, + rather than masking through the authority's end ([built-in tools](../reference/shared-core/built-in-tools.md#finite-diagnostic-credential-masking)). - **Split and normalized forms.** Every rule except the literal query names also probes the terminal-normalized text; a credential revealed only by normalization masks the **whole** - diagnostic. The key prefixes, URL - userinfo and the exact header names are additionally probed in the LF/TAB-joined view at - original-text boundaries, and a credential revealed only by such a split masks the whole + diagnostic. The key prefixes, URL userinfo and the exact header names are additionally probed in + the LF/TAB-joined view at original-text boundaries — of the raw text too when terminal + normalization changed it, since a terminal sequence there may take a split credential's first + letter (ESC `A` before `AIza…`) — and a credential revealed only by such a split masks the whole diagnostic too. A `Bearer`/`Basic` label may itself be split by LF/TAB. - **Documented trade-offs, all failing closed.** A scheme glued to a word character is masked in place; an unlisted scheme ending in a listed one (`gftp://`, `jdbc:mysql://`) matches by its - suffix; prose that the whitespace-stripped residual probe glues to a path-less URL and a later - `@` blanks the whole diagnostic, and so does a key prefix (`sk-`, `AIza`) that the same probe - joins with the following words to its minimum length, at the start of the text or, since the - scrub repeats, right after another rule's mask (`Authorization: Bearer xyz sk-ant api key - invalid`); a host fragment with no assignment (`https://host.example#sec@x`), - a single-label host query and an over-long host-shaped prefix are masked, and so is the text - after the last `@` of such an ambiguous authority unless it is host-shaped; a - `Bearer`/`Basic` label that ends a line takes the next token with it; an opaque value after a - label ends only at whitespace, so in compact JSON it masks to the end of the run; nine or more - concatenated labels, read in the raw text as well as the normalized one, and a token that begins with `,`, `;` or `}` after a `Bearer`/`Basic` - label in a header value, blank the whole diagnostic. Past eight separate labels after a header - name, and in a value of labels alone glued behind punctuation (`Bearer |Bearer`), the outcome - depends on what precedes the header name: when its nearest non-whitespace character is a word - character, `-` or `_`, the diagnostic stays in place (the last label masks any token); - otherwise, the start of the text included, it is blanked. No token is visible either way, and - the output is stable under a second scrub. -- **Limits.** Userinfo under an unlisted scheme is not masked, including driver-qualified - spellings such as `postgresql+psycopg2://`. Opaque `Bearer`/`Basic` values, `api-key` header - values and literal secret query values end at any whitespace, LF and TAB included, so a - fragment after such a break stays visible, as it does after an ordinary space; a label glued to - a quoted token without whitespace (`Bearer""`) is likewise outside the shape, as is a - token after a JSON escape (`\n`, `\t`) that stands for the whitespace after a label. A word - glued before `sftp://` on the same line (`statusftp://user@h`) reads as `sftp`, so its colonless - userinfo stays visible like any `sftp` user: every whitespace-joined probe, the CLI's display - residual among them, glues prose to a scheme the same way, and reading it as `ftp` would blank - an ordinary `connect to sftp://user@host` diagnostic. (A word that ends a line before `ftp://` - is no such glue: an LF/TAB-split ftp credential after it blanks the diagnostic.) Malformed URLs - (`https:/u:p@h`), a raw `/` inside a password, invisible characters inside a scheme and JSON - whose quotes are themselves escaped as `\u0022` are outside the finite shapes. In a JSON-encoded + suffix; prose that the whitespace-stripped residual probe glues to a path-less URL and a later `@` + blanks the whole diagnostic, and so does a key prefix (`sk-`, `AIza`) that the same probe joins + with the following words to its minimum length, at the start of the text or, since the scrub + repeats, right after another rule's mask (`Authorization: Bearer xyz sk-ant api key invalid`); a + path-less terminal hyperlink target (OSC 8) that the raw LF/TAB-joined probe joins with a later + `@` (a hyperlinked table cell before an e-mail address) blanks it as well; a host fragment with no + assignment (`https://host.example#sec@x`), a single-label host query and an over-long host-shaped + prefix are masked, and so is the text after the last `@` of such an ambiguous authority unless it + is host-shaped; a `Bearer`/`Basic` label that ends a line takes the next token with it; an opaque + value after a label ends only at whitespace, so in compact JSON it masks to the end of the run; + nine or more concatenated labels, read in the raw text as well as the normalized one, and a token + that begins with `,`, `;` or `}` after a `Bearer`/`Basic` label in a header value, blank the whole + diagnostic. Past eight separate labels after a header name, and in a value of labels alone glued + behind punctuation (`Bearer |Bearer`), the outcome depends on what precedes the header name: when + its nearest non-whitespace character is a word character, `-` or `_`, the diagnostic stays in + place (the last label masks any token); otherwise, the start of the text included, it is blanked. + No token is visible either way, and the output is stable under a second scrub. +- **Limits.** Userinfo under an unlisted scheme is not masked, including driver-qualified spellings + such as `postgresql+psycopg2://`. Opaque `Bearer`/`Basic` values, `api-key` header values and + literal secret query values end at any whitespace, LF and TAB included, so a fragment after such a + break stays visible, as it does after an ordinary space; a label glued to a quoted token without + whitespace (`Bearer""`) is likewise outside the shape, as is a token after a JSON escape + (`\n`, `\t`) that stands for the whitespace after a label. A JSON escape right after a header + name's colon starts its token, so a value glued to the escape is masked with the escape's letter + (`"x-api-key":"\n"` → `"x-api-key":"\[REDACTED]"`), while a quoted value behind such an escape + stays visible (`x-api-key:\t""`). A terminal sequence with a parameter whose final byte is a + shape's first letter (ESC `[` `1` `B` before `earer `), and a terminated device-control, + start-of-string or application-program string (ESC `P`, ESC `X` or ESC `_`, ended by ST or BEL) + glued before a shape, hide that shape from every view: in the raw text the sequence's last byte + glues to the shape's first letter, and normalization removes it. A word glued before `sftp://` on + the same line (`statusftp://user@h`) reads as `sftp`, so its colonless userinfo stays visible like + any `sftp` user: every whitespace-joined probe, the CLI's display residual among them, glues prose + to a scheme the same way, and reading it as `ftp` would blank an ordinary + `connect to sftp://user@host` diagnostic. (A word that ends a line before `ftp://` is no such + glue: an LF/TAB-split ftp credential after it blanks the diagnostic.) Malformed URLs + (`https:/u:p@h`), a raw `/` inside a password, invisible characters inside a scheme and JSON whose + quotes are themselves escaped as `\u0022` are outside the finite shapes. In a JSON-encoded diagnostic, an escape written as two characters (`\n`, `\t`, `\r`) and glued before a key prefix, a header name or a label defeats that shape's word boundary, so the credential after it stays visible (`"key rejected:\nsk-…"`). Core's journal projection shares this limit for key prefixes - and scheme labels; its key/value rule's word-character prefix still reads a header or secret-key - name glued after such an escape (`"err:\napi-key: …"`) and masks the value with it. A header value + and scheme labels, while core's summary also reads each such escape as its separator and masks the + credential after it; core's key/value rule's word-character prefix still reads a header or + secret-key name glued after such an escape (`"err:\napi-key: …"`) and masks the value with it. A header value that begins with `,`, `;` or `}` and has no label before it is an empty token, so the text after it stays visible. The backstop has no generic key/value rule: a secret key inside a URL authority whose value holds `@` — after `;` (`https://svc;a=1;password=P@ss…`), after `&` unless it is one diff --git a/packages/core/src/tools/bounding.test.ts b/packages/core/src/tools/bounding.test.ts index bc1ad4fd..3406b9c0 100644 --- a/packages/core/src/tools/bounding.test.ts +++ b/packages/core/src/tools/bounding.test.ts @@ -24,6 +24,13 @@ vi.mock('./fixed-point.js', async (importOriginal) => { }; }); +/** + * Eight-MiB cases take well under a second. Their bound is this case deadline, far below the minutes a + * quadratic scan would need, rather than a wall-clock assertion that a loaded runner can trip. It is + * also the largest case budget the secrets causal-control phases admit (`CAUSAL_TIMEOUT_BUDGET`). + */ +const ADMITTED_SIZE_CASE_TIMEOUT_MS = 15_000; + // The observation must not keep every admitted-size input and projection alive for the whole file. afterEach(() => { vi.mocked(fixedPointOf).mockClear(); @@ -159,6 +166,20 @@ describe('boundForModel', () => { const bounded = await boundForModel(undefined, TINY, host()); expect(bounded.truncated).toBe(false); expect(bounded.value).toBeUndefined(); + expect(bounded.summary).toBe(''); + }); + + it('summarizes a result that has no JSON form as unserializable, without rejecting', async () => { + const throwing = Object.defineProperty({}, 'body', { + enumerable: true, + get() { + throw new Error('getter fails'); + }, + }); + for (const result of [{ n: 10n }, throwing]) { + const bounded = await boundForModel(result, BIG, host()); + expect(bounded.summary).toBe('[unserializable]'); + } }); it('renders a circular result without throwing (the cycle-safe redaction walk breaks the cycle)', async () => { @@ -1025,15 +1046,17 @@ describe('PR95 B2 projection boundaries and admitted-scale controls', () => { }, ); - it('bounds admitted-size failed finite header preparation', () => { - const fragment = 'wordx-\tapi-\nkey '; - const input = fragment.repeat(Math.floor((8 * 1024 * 1024 - 256) / fragment.length)); - const started = performance.now(); - const output = redactSecretShapedText(input); - expect(output.length, 'SECRETS_B2_HEADER_PREPARATION').toBe(input.length); - expect(output === input, 'SECRETS_B2_HEADER_PREPARATION').toBe(true); - expect(performance.now() - started, 'SECRETS_B2_HEADER_PREPARATION').toBeLessThan(5_000); - }); + it( + 'bounds admitted-size failed finite header preparation', + () => { + const fragment = 'wordx-\tapi-\nkey '; + const input = fragment.repeat(Math.floor((8 * 1024 * 1024 - 256) / fragment.length)); + const output = redactSecretShapedText(input); + expect(output.length, 'SECRETS_B2_HEADER_PREPARATION').toBe(input.length); + expect(output === input, 'SECRETS_B2_HEADER_PREPARATION').toBe(true); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); it.each([ ['sk-proj-', '\n'], @@ -1098,41 +1121,47 @@ describe('PR95 B2 projection boundaries and admitted-scale controls', () => { } }); - it('bounds admitted-size repeated invalid JWT starts without losing later valid spans', () => { - const fragment = `eyJ${'A'.repeat(61)}\n`; - const publicPrefix = fragment.repeat(Math.floor((8 * 1024 * 1024 - 256) / fragment.length)); - const started = performance.now(); - // No dot, or a huge second run with no second dot: both full JWT attempts would scan to EOF. - for (const suffix of ['', `.${'B'.repeat(128)}`]) { - const input = `${publicPrefix}${suffix}; sk-proj-${'C'.repeat(32)}\t${'D'.repeat(32)}; public=A`; - const expected = `${publicPrefix}${suffix}; [redacted]; public=A`; + it( + 'bounds admitted-size repeated invalid JWT starts without losing later valid spans', + () => { + const fragment = `eyJ${'A'.repeat(61)}\n`; + const publicPrefix = fragment.repeat(Math.floor((8 * 1024 * 1024 - 256) / fragment.length)); + // No dot, or a huge second run with no second dot: both full JWT attempts would scan to EOF. + for (const suffix of ['', `.${'B'.repeat(128)}`]) { + const input = `${publicPrefix}${suffix}; sk-proj-${'C'.repeat(32)}\t${'D'.repeat(32)}; public=A`; + const expected = `${publicPrefix}${suffix}; [redacted]; public=A`; + const output = redactSecretShapedText(input); + expect(output.length, 'SECRETS_B2_BOUNDED_PROBE').toBe(expected.length); + expect(output === expected, 'SECRETS_B2_BOUNDED_PROBE').toBe(true); + } + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); + + it( + 'bounds admitted-size raw hyphen-separated invalid JWT starts and masks a later whole key', + () => { + const fragment = 'eyJAAAAAAAAAA-'; + const publicPrefix = fragment.repeat(Math.floor((8 * 1024 * 1024 - 256) / fragment.length)); + const input = `${publicPrefix}sk-proj-${'C'.repeat(32)}-${'D'.repeat(32)}; public=A`; + const expected = `${publicPrefix}[redacted]; public=A`; const output = redactSecretShapedText(input); - expect(output.length, 'SECRETS_B2_BOUNDED_PROBE').toBe(expected.length); - expect(output === expected, 'SECRETS_B2_BOUNDED_PROBE').toBe(true); - } - expect(performance.now() - started, 'SECRETS_B2_BOUNDED_PROBE').toBeLessThan(5_000); - }); - - it('bounds admitted-size raw hyphen-separated invalid JWT starts and masks a later whole key', () => { - const fragment = 'eyJAAAAAAAAAA-'; - const publicPrefix = fragment.repeat(Math.floor((8 * 1024 * 1024 - 256) / fragment.length)); - const input = `${publicPrefix}sk-proj-${'C'.repeat(32)}-${'D'.repeat(32)}; public=A`; - const expected = `${publicPrefix}[redacted]; public=A`; - const started = performance.now(); - const output = redactSecretShapedText(input); - expect(output.length, 'SECRETS_B2_RAW_BOUNDED_PROBE').toBe(expected.length); - expect(output === expected, 'SECRETS_B2_RAW_BOUNDED_PROBE').toBe(true); - expect(performance.now() - started, 'SECRETS_B2_RAW_BOUNDED_PROBE').toBeLessThan(5_000); - }); - - it('preserves dense admitted-size public boundaries with bounded preparation', () => { - const started = performance.now(); - for (const fragment of ['a\n', 'a ']) { - const input = `${fragment.repeat(4 * 1024 * 1024 - 1)}\n`; - expect(redactSecretShapedText(input) === input, 'SECRETS_B2_BOUNDED_PROBE').toBe(true); - } - expect(performance.now() - started, 'SECRETS_B2_BOUNDED_PROBE').toBeLessThan(5_000); - }); + expect(output.length, 'SECRETS_B2_RAW_BOUNDED_PROBE').toBe(expected.length); + expect(output === expected, 'SECRETS_B2_RAW_BOUNDED_PROBE').toBe(true); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); + + it( + 'preserves dense admitted-size public boundaries with bounded preparation', + () => { + for (const fragment of ['a\n', 'a ']) { + const input = `${fragment.repeat(4 * 1024 * 1024 - 1)}\n`; + expect(redactSecretShapedText(input) === input, 'SECRETS_B2_BOUNDED_PROBE').toBe(true); + } + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); it('retains prefix evidence inside a header and preserves the same public tail after multiple masks', () => { const credential = `sk-proj-${'A'.repeat(24)}\n${'B'.repeat(24)}`; @@ -1274,18 +1303,22 @@ describe('PR95 raw ?/# userinfo in the journal projection', () => { expect(redactSecretShapedText(upper)).toBe(upper); }); - it('masks an admitted-size dotted userinfo prefix without exhausting the RegExp stack', () => { - const labels = 'a.'.repeat(Math.floor((8 * 1024 * 1024 - 256 - 64) / 2)); - const input = `upstream said https://${labels}a?x=1@h.example failed`; - let output = ''; - expect(() => { - output = redactSecretShapedText(input); - }, 'SECRETS_B2_CORE_USERINFO_HOST_LENGTH').not.toThrow(); - expect( - output === 'upstream said https://[redacted]@h.example failed', - 'SECRETS_B2_CORE_USERINFO_HOST_LENGTH', - ).toBe(true); - }, 15_000); + it( + 'masks an admitted-size dotted userinfo prefix without exhausting the RegExp stack', + () => { + const labels = 'a.'.repeat(Math.floor((8 * 1024 * 1024 - 256 - 64) / 2)); + const input = `upstream said https://${labels}a?x=1@h.example failed`; + let output = ''; + expect(() => { + output = redactSecretShapedText(input); + }, 'SECRETS_B2_CORE_USERINFO_HOST_LENGTH').not.toThrow(); + expect( + output === 'upstream said https://[redacted]@h.example failed', + 'SECRETS_B2_CORE_USERINFO_HOST_LENGTH', + ).toBe(true); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); }); describe('PR95 review round 4: ambiguous authorities, URL schemes after labels and split edges', () => { @@ -1308,18 +1341,22 @@ describe('PR95 review round 4: ambiguous authorities, URL schemes after labels a } }); - it('masks an admitted-size query value inside an ambiguous userinfo', () => { - const labels = 'a.'.repeat(Math.floor((8 * 1024 * 1024 - 256 - 64) / 2)); - const input = `upstream said https://user:pw?x=1@${labels}a failed`; - let output = ''; - expect(() => { - output = redactSecretShapedText(input); - }, 'SECRETS_B5_CORE_QUERY_EXTENT').not.toThrow(); - expect( - output === 'upstream said https://[redacted] failed', - 'SECRETS_B5_CORE_QUERY_EXTENT', - ).toBe(true); - }, 15_000); + it( + 'masks an admitted-size query value inside an ambiguous userinfo', + () => { + const labels = 'a.'.repeat(Math.floor((8 * 1024 * 1024 - 256 - 64) / 2)); + const input = `upstream said https://user:pw?x=1@${labels}a failed`; + let output = ''; + expect(() => { + output = redactSecretShapedText(input); + }, 'SECRETS_B5_CORE_QUERY_EXTENT').not.toThrow(); + expect( + output === 'upstream said https://[redacted] failed', + 'SECRETS_B5_CORE_QUERY_EXTENT', + ).toBe(true); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); it('masks the userinfo of a URL behind a scheme label, split or not, and a token glued to a URL', () => { for (const [input, expected] of [ @@ -1483,14 +1520,18 @@ describe('PR95 review round 5: an ambiguous userinfo masked with exactly the val }, ); - it('masks a token glued to a URL at admitted size', () => { - const input = `Bearer ${'a'.repeat(8 * 1024 * 1024 - 64)}://h.example`; - let output = ''; - expect(() => { - output = redactSecretShapedText(input); - }, 'SECRETS_B5_CORE_SCHEME_ORDER').not.toThrow(); - expect(output === 'Bearer [redacted]://h.example', 'SECRETS_B5_CORE_SCHEME_ORDER').toBe(true); - }, 15_000); + it( + 'masks a token glued to a URL at admitted size', + () => { + const input = `Bearer ${'a'.repeat(8 * 1024 * 1024 - 64)}://h.example`; + let output = ''; + expect(() => { + output = redactSecretShapedText(input); + }, 'SECRETS_B5_CORE_SCHEME_ORDER').not.toThrow(); + expect(output === 'Bearer [redacted]://h.example', 'SECRETS_B5_CORE_SCHEME_ORDER').toBe(true); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); }); describe('PR95 review round 6: one merged pass of every core rule', () => { @@ -1695,50 +1736,60 @@ describe('PR95 review round 6: one merged pass of every core rule', () => { ).toEqual({ url: 'x\thttps://[redacted]' }); }); - it('keeps many query assignments in an ambiguous userinfo linear at admitted size', () => { - const size = 8 * 1024 * 1024 - 256; - const run = (prefix: string, unit: string, suffix: string) => - prefix + - unit.repeat(Math.floor((size - prefix.length - suffix.length) / unit.length)) + - suffix; - for (const [input, expected] of [ - [run('https://x', '?password=', '@h'), 'https://[redacted]'], - [run('https://x', '?a=', '@h'), 'https://[redacted]'], - [run('https://svc?', '#password=x', '@h'), 'https://[redacted]'], - [run('see https://svc', '?token=a@b', ''), 'see https://[redacted]'], - [run('redis://app:pw', '?client_secret=x', '@db'), 'redis://[redacted]'], - ] as const) { - let output = ''; - expect(() => { - output = redactSecretShapedText(input); - }, 'SECRETS_B6_CORE_MERGE').not.toThrow(); - expect(output === expected, 'SECRETS_B6_CORE_MERGE').toBe(true); - } - }, 15_000); - - it('keeps nested values, labels and open PEM blocks linear at admitted size', () => { - const size = 8 * 1024 * 1024 - 256; - const count = (unit: string) => Math.floor(size / unit.length); - const keys = 'token=ab>'.repeat(count('token=ab>')); - expect(redactSecretShapedText(keys) === '[redacted]', 'SECRETS_B6_CORE_NESTED_KEY').toBe(true); - const labels = count('Bearer abcdefgh/'); - expect( - redactSecretShapedText('Bearer abcdefgh/'.repeat(labels)) === - `Bearer${' [redacted]'.repeat(labels)}`, - 'SECRETS_B6_CORE_NESTED_LABEL', - ).toBe(true); - // Only the BEGIN marker whose body reaches the far END within the body bound opens a block; the - // markers before it stay, and each of them reuses one END search instead of scanning to it again. - const marker = '-----BEGIN PRIVATE KEY-----\n'; - const open = marker.repeat(count(marker) - 1); - const first = - Math.ceil((open.length - 20_000 - (marker.length - 1)) / marker.length) * marker.length; - expect( - redactSecretShapedText(`password=hunter2secret ${open}-----END PRIVATE KEY-----`) === - `[redacted] ${open.slice(0, first)}[redacted]`, - 'SECRETS_B6_CORE_MERGE', - ).toBe(true); - }, 15_000); + it( + 'keeps many query assignments in an ambiguous userinfo linear at admitted size', + () => { + const size = 8 * 1024 * 1024 - 256; + const run = (prefix: string, unit: string, suffix: string) => + prefix + + unit.repeat(Math.floor((size - prefix.length - suffix.length) / unit.length)) + + suffix; + for (const [input, expected] of [ + [run('https://x', '?password=', '@h'), 'https://[redacted]'], + [run('https://x', '?a=', '@h'), 'https://[redacted]'], + [run('https://svc?', '#password=x', '@h'), 'https://[redacted]'], + [run('see https://svc', '?token=a@b', ''), 'see https://[redacted]'], + [run('redis://app:pw', '?client_secret=x', '@db'), 'redis://[redacted]'], + ] as const) { + let output = ''; + expect(() => { + output = redactSecretShapedText(input); + }, 'SECRETS_B6_CORE_MERGE').not.toThrow(); + expect(output === expected, 'SECRETS_B6_CORE_MERGE').toBe(true); + } + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); + + it( + 'keeps nested values, labels and open PEM blocks linear at admitted size', + () => { + const size = 8 * 1024 * 1024 - 256; + const count = (unit: string) => Math.floor(size / unit.length); + const keys = 'token=ab>'.repeat(count('token=ab>')); + expect(redactSecretShapedText(keys) === '[redacted]', 'SECRETS_B6_CORE_NESTED_KEY').toBe( + true, + ); + const labels = count('Bearer abcdefgh/'); + expect( + redactSecretShapedText('Bearer abcdefgh/'.repeat(labels)) === + `Bearer${' [redacted]'.repeat(labels)}`, + 'SECRETS_B6_CORE_NESTED_LABEL', + ).toBe(true); + // Only the BEGIN marker whose body reaches the far END within the body bound opens a block; the + // markers before it stay, and each of them reuses one END search instead of scanning to it again. + const marker = '-----BEGIN PRIVATE KEY-----\n'; + const open = marker.repeat(count(marker) - 1); + const first = + Math.ceil((open.length - 20_000 - (marker.length - 1)) / marker.length) * marker.length; + expect( + redactSecretShapedText(`password=hunter2secret ${open}-----END PRIVATE KEY-----`) === + `[redacted] ${open.slice(0, first)}[redacted]`, + 'SECRETS_B6_CORE_MERGE', + ).toBe(true); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); it('masks a PEM block whose body reaches exactly the body bound', () => { const begin = '-----BEGIN PRIVATE KEY-----'; @@ -1786,6 +1837,18 @@ describe('PR95 review round 7: userinfo ownership, admitted quoted values, glued // A username-shaped key on the next line is no username of this URL. ['see https://\nDB_PASSWORD:P@ssw0rd123 now', 'see https://[redacted] now', 'ssw0rd123'], ['x https://\tSERVICE_TOKEN:Adm1n@2024x', 'x https://[redacted]', '2024x'], + // A mask past the userinfo's @ stops at the next line, which is read on its own. + [ + 'DATABASE_URL=https://db.internal.example.com\nDB_PASSWORD=P@ssw0rd123\nNEXT=1', + 'DATABASE_URL=https://[redacted]\nNEXT=1', + 'ssw0rd123', + ], + // Read on its own line, the key masks its whole value, a host-shaped tail included. + [ + 'see https://\nDB_PASSWORD=P@host.example now', + 'see https://\n[redacted] now', + 'host.example', + ], ]); }); @@ -1803,10 +1866,47 @@ describe('PR95 review round 7: userinfo ownership, admitted quoted values, glued '{"content":"REDIS_URL=redis://[redacted]"}', 'Passw0rd', ], - // A colon after a raw ?/#, or before any username, marks no user. + // Its value runs to the key rule's own delimiter, through the host, path and query. + [ + 'https://user:password=Secr3tValue1@host.example/v1/a', + 'https://[redacted]', + 'host.example', + ], + [ + 'GET https://svc:secret=Qz7abc@api.example.com/v1/items/123?expand=all&page=2 200', + 'GET https://[redacted]&page=2 200', + 'api.example.com', + ], + ['sqlserver://;password=Adm1n@2024.prod/app', 'sqlserver://;[redacted]', '2024.prod'], + [ + '"custom\t://user:db.password=Qz5SjmRYV4@Qz7uzLd396yJU1\neyJQz4kX2Nk3X2tw9s.Qz6CteaZX56zE1JL09cObAqM06.Qz0wvWBdMRLWj7 x', + '"custom\t://[redacted]\n[redacted] x', + 'Qz7uzLd396yJU1', + ], + ]); + // From a raw ?/# on, a key reads as a query parameter whose value runs to its own end too, and a + // colon after a raw ?/# or before any username marks no user. + projects('SECRETS_B7_CORE_USERNAME_ZONE', [ ['https://password=abcdefgh?x:y@host.example/path', 'https://[redacted]', 'abcdefgh'], ['https://:password=abcdefghij?x@host.example/p', 'https://[redacted]', 'abcdefghij'], ]); + // Only a value some rule reads runs to its own end: a short value under another name is masked + // through the userinfo's last '@', and its text after that '@' reads as the host. + expect( + redactSecretShapedText('https://svc?a=1&code=Adm1n@2024prod'), + 'SECRETS_B7_CORE_USERNAME_ZONE', + ).toBe('https://[redacted]@2024prod'); + // Such a value is a recognized credential: calls that differ only in its host or path share a + // journal projection, while a URL whose userinfo holds no key keeps them apart. + const identity = (url: string) => projectSecretShapedValue({ url, method: 'POST' }).value; + expect( + identity('https://user:password=Secr3tValue1@host.example/v1/a'), + 'SECRETS_B7_CORE_USERNAME_ZONE', + ).toEqual(identity('https://user:password=Secr3tValue1@other.example/v2/delete/7')); + expect( + identity('https://user:Secr3tValue1@host.example/v1/a'), + 'SECRETS_B7_CORE_USERNAME_ZONE', + ).not.toEqual(identity('https://user:Secr3tValue1@host.example/v1/b')); }); it('keeps the host when only the username names a secret, split after its colon or not', () => { @@ -1824,22 +1924,33 @@ describe('PR95 review round 7: userinfo ownership, admitted quoted values, glued ).toBe('https://[redacted]'); }); - it('projects exactly admitted-size quoted values without exhausting the RegExp stack', () => { - const size = 8 * 1024 * 1024; - for (const [input, expected] of [ - [`token="${'a'.repeat(size - 7)}`, undefined], - [`password="${'a'.repeat(size - 11)}"`, '[redacted]'], - [`password='${'a'.repeat(size - 11)}'`, '[redacted]'], - [`{"access_token":"${'a'.repeat(size - 19)}"}`, '{"[redacted]}'], - ] as const) { - let output = ''; - expect(() => { - output = redactSecretShapedText(input); - }, 'SECRETS_B7_CORE_QUOTED_SCAN').not.toThrow(); - // An unclosed quote is no value: the text stays as it is. - expect(output === (expected ?? input), 'SECRETS_B7_CORE_QUOTED_SCAN').toBe(true); - } - }, 15_000); + it( + 'projects exactly admitted-size quoted values without exhausting the RegExp stack', + () => { + const size = 8 * 1024 * 1024; + const exact = (prefix: string, suffix: string) => + `${prefix}${'a'.repeat(size - prefix.length - suffix.length)}${suffix}`; + for (const [input, expected] of [ + // An unclosed value is read as an unquoted run from after its quote; here that run reaches the + // end of the text. + [exact('token="', ''), '[redacted]'], + [exact('password="', '"'), '[redacted]'], + [exact("password='", "'"), '[redacted]'], + [exact('{"access_token":"', '"}'), '{"[redacted]}'], + // One JSON level down, closed and unclosed. + [exact('{"b":"{\\"token\\":\\"', '\\"}"}'), '{"b":"{\\"[redacted]}"}'], + [exact('{"b":"token=\\"', '"}'), '{"b":"[redacted]"}'], + ] as const) { + expect(input.length, 'SECRETS_B7_CORE_QUOTED_SCAN').toBe(size); + let output = ''; + expect(() => { + output = redactSecretShapedText(input); + }, 'SECRETS_B7_CORE_QUOTED_SCAN').not.toThrow(); + expect(output === expected, 'SECRETS_B7_CORE_QUOTED_SCAN').toBe(true); + } + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); it("keeps quoted values on one line and pins the key rule's names and delimiters", () => { const keys = (value: unknown) => expect(value, 'SECRETS_B7_CORE_KEY_PIECES'); @@ -1857,20 +1968,31 @@ describe('PR95 review round 7: userinfo ownership, admitted quoted values, glued '[redacted]&next=public', ); keys(redactSecretShapedText('token=abcdefgh,next')).toBe('[redacted],next'); - // A quoted value never crosses a line break, and a key must start within its word's first 33 characters. + keys(redactSecretShapedText("token=abcdefgh'tail")).toBe("[redacted]'tail"); + // A quoted value closes on its line. One that does not, even past an escaped line break, is read + // as an unquoted run from after its quote, to that line's end at most: its quote may instead close + // a string the key sits in. for (const kept of [ 'password="abc\ndef" tail', "password='abc\rdef' tail", - `x${'y'.repeat(40)}token=listed`, + 'password="abc\ndef tail', + 'password="abc\\\ndef tail', ]) { keys(redactSecretShapedText(kept)).toBe(kept); } - }); - - it('masks a run of glued fixed-length ids in one pass, split or not', async () => { - const ids = 'AKIAAAAAAAAAAAAAAAAAASIABBBBBBBBBBBBBBBBAKIACCCCCCCCCCCCCCCCASIADDDDDDDDDDDDDDDD'; - projects('SECRETS_B7_CORE_CHAIN', [ - [`${ids} public tail`, `${'[redacted]'.repeat(4)} public tail`, 'DDDDDDDD'], + keys(redactSecretShapedText('password="abcdef\nghi" tail')).toBe('[redacted]\nghi" tail'); + keys(redactSecretShapedText('password="abcdef\\\nghi tail')).toBe('[redacted]\nghi tail'); + // A key must start within its word's first 33 characters. + const listed = `x${'y'.repeat(40)}token=listed`; + keys(redactSecretShapedText(listed)).toBe(listed); + }); + + it('recognizes a credential glued to another one pass later and redacts a run that never settles', async () => { + const two = 'AKIAAAAAAAAAAAAAAAAAASIABBBBBBBBBBBBBBBB'; + projects('SECRETS_B8_CORE_GLUED', [ + [`${two} public tail`, '[redacted][redacted] public tail', 'BBBBBBBB'], + // Four glued ids need four passes: the text does not settle and fails closed. + [`${two}AKIACCCCCCCCCCCCCCCCASIADDDDDDDDDDDDDDDD public tail`, '[redacted]', 'DDDDDDDD'], [ 'AKIAABCDEFGHIJKLMNOPghp_AAAAAAAAAAAA\nBBBBBBBBBBBBBBBB done', '[redacted][redacted] done', @@ -1881,13 +2003,35 @@ describe('PR95 review round 7: userinfo ownership, admitted quoted values, glued '[redacted][redacted] done', 'BBBBBBBB', ], + // A glued credential is read at the mask's boundary, so it neither cuts a wider credential + // that starts inside it nor swallows a key name or a scheme label after it. + [ + 'AKIAIOSFODNN7EXAMPLExoxb-eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0In0.SflKxwRJSMeKKF2QT4fwpM', + '[redacted]xoxb-[redacted]', + 'SflKxwRJSMeKKF2QT4fwpM', + ], + [ + 'AKIAIOSFODNN7EXAMPLExoxb-sk-proj-abcdefghij_klmnopqrstuv', + '[redacted]xoxb-[redacted]', + 'klmnopqrstuv', + ], + [ + 'AKIAIOSFODNN7EXAMPLEnpm_abcdefghij0123456789token=SuperSecret99', + '[redacted][redacted]', + 'SuperSecret99', + ], + [ + 'AKIAABCDEFGHIJKLMNOPsk-AKIAABCDEFGHIJKLMNOPBearer abcdefghijklmnop', + '[redacted]sk-[redacted]Bearer [redacted]', + 'abcdefghijklmnop', + ], ]); const summary = await boundForModel( 'AKIAABCDEFGHIJKLMNOPsk-proj-AAAAAAAAAAAAAAAA\tBBBBBBBBBBBB done', BIG, {}, ); - expect(summary.summary, 'SECRETS_B7_CORE_CHAIN').toBe('[redacted][redacted] done'); + expect(summary.summary, 'SECRETS_B8_CORE_GLUED').toBe('[redacted][redacted] done'); }); it("summarizes with the projection's fixed point before one-line normalization", async () => { @@ -1929,16 +2073,20 @@ describe('PR95 review round 7: userinfo ownership, admitted quoted values, glued it('reports whether a journal projection settled', () => { const journal = (value: unknown) => expect(value, 'SECRETS_B7_CORE_JOURNAL_SETTLED'); - const content = - 'AKIAAAAAAAAAAAAAAAAAASIABBBBBBBBBBBBBBBBAKIACCCCCCCCCCCCCCCCASIADDDDDDDDDDDDDDDD\nexpect(scan(x)).toBe(4)\n'; - journal(projectSecretShapedValue({ path: 'test/fixtures/keys.txt', content })).toEqual({ - value: { - path: 'test/fixtures/keys.txt', - content: `${'[redacted]'.repeat(4)}\nexpect(scan(x)).toBe(4)\n`, - }, + const ids = 'AKIAAAAAAAAAAAAAAAAAASIABBBBBBBBBBBBBBBB'; + journal( + projectSecretShapedValue({ path: 'test/fixtures/keys.txt', content: `${ids}\nscan(x)\n` }), + ).toEqual({ + value: { path: 'test/fixtures/keys.txt', content: '[redacted][redacted]\nscan(x)\n' }, settled: true, }); - // A string whose projection does not settle projects to the mask and is reported. + // Four glued ids need four passes, and a key whose short value names the next key one per key. + for (const content of [`${ids}${ids}\nscan(x)\n`, 'token:token =token ://?-=Q']) { + journal(projectSecretShapedValue({ path: 'p', content })).toEqual({ + value: { path: 'p', content: '[redacted]' }, + settled: false, + }); + } vi.mocked(fixedPointOf).mockImplementationOnce(() => undefined); journal(projectSecretShapedValue('drifting text')).toEqual({ value: '[redacted]', @@ -1946,3 +2094,721 @@ describe('PR95 review round 7: userinfo ownership, admitted quoted values, glued }); }); }); + +describe('PR95 review round 8: structured summaries, escaped and unclosed quotes, continuation labels', () => { + const projects = (marker: string, pairs: readonly (readonly [string, string, string])[]) => { + for (const [input, expected, secret] of pairs) { + const output = redactSecretShapedText(input); + expect(output, marker).toBe(expected); + expect(output.includes(secret), marker).toBe(false); + expect(redactSecretShapedText(output), marker).toBe(output); + } + }; + + it('summarizes a structured result from its JSON once credential members are masked', async () => { + for (const [result, secrets] of [ + [ + { content: 'DB_PASSWORD="hunter2hunter2"\nNODE_ENV=production\n', mimeType: 'text/plain' }, + ['hunter2hunter2'], + ], + [ + { + content: + '{\n "database": {"password": "Qz9correcthorse"}, "client_secret": "Qz5clientsecretval"}', + }, + ['Qz9correcthorse', 'Qz5clientsecretval'], + ], + [ + { + status: 200, + headers: {}, + body: '{"access_token":"2YotnFZFEjr1zCsicMWpAA","refresh_token":"tGzv3JOkF0XG5Qx2TlKWIA"}', + }, + ['2YotnFZFEjr1zCsicMWpAA', 'tGzv3JOkF0XG5Qx2TlKWIA'], + ], + [ + { + exitCode: 0, + stdout: 'export AWS_SECRET_ACCESS_KEY="wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY"\n', + stderr: '', + }, + ['wJalrXUtnFEMIK7MDENG'], + ], + // The walk also masks what no text rule reads: a secret-named key's whole value and the + // element after a credential flag. + [{ token: { value: 'Qz7nestedSecret' } }, ['Qz7nestedSecret']], + [{ argv: ['--password', 'Qz7flagSecret'] }, ['Qz7flagSecret']], + ] as const) { + const bounded = await boundForModel(result, BIG, {}); + expect(bounded.value, 'SECRETS_B8_CORE_SUMMARY_WALK').toBe(result); + for (const secret of secrets) { + expect(bounded.summary.includes(secret), 'SECRETS_B8_CORE_SUMMARY_WALK').toBe(false); + } + } + const summary = await boundForModel( + { content: 'DB_PASSWORD="hunter2hunter2"\nNODE_ENV=production\n' }, + BIG, + {}, + ); + expect(summary.summary, 'SECRETS_B8_CORE_SUMMARY_WALK').toBe( + '{"content":"[redacted]\\nNODE_ENV=production\\n"}', + ); + }); + + it('reads a quoted value one JSON string down', () => { + projects('SECRETS_B8_CORE_ESCAPED_QUOTE', [ + [ + '{"body":"{\\"password\\":\\"hunter2hunter2\\",\\"user\\":\\"bob\\"}"}', + '{"body":"{\\"[redacted],\\"user\\":\\"bob\\"}"}', + 'hunter2hunter2', + ], + [ + '{"stdout":"export TOKEN=\\"Qz7abc12345\\" && run"}', + '{"stdout":"export [redacted] && run"}', + 'Qz7abc12345', + ], + [ + '{"h":"{\\"x-api-key\\":\\"Bearer Qz7abcdefghijkl\\"}"}', + '{"h":"{\\"[redacted]}"}', + 'Qz7abcdefghijkl', + ], + // An escaped tab or line break may stand between the separator and the escaped quote. + ['{"cfg":"password:\\t\\"hunter2Secr3t\\""}', '{"cfg":"[redacted]"}', 'hunter2Secr3t'], + ['{"cfg":"password:\\n \\"hunter2Secr3t\\""}', '{"cfg":"[redacted]"}', 'hunter2Secr3t'], + // A raw tab may stand among them, an escaped CR LF is one line break, and an escaped apostrophe + // opens such a value too; so does a raw one, which JSON leaves unescaped. + ['{"cfg":"password:\\n\t\\"hunter2Secr3t\\""}', '{"cfg":"[redacted]"}', 'hunter2Secr3t'], + ['{"cfg":"password:\\r\\n\\"hunter2Secr3t\\""}', '{"cfg":"[redacted]"}', 'hunter2Secr3t'], + ['{"cfg":"password:\\n \\\'hunter2Secr3t\\\'"}', '{"cfg":"[redacted]"}', 'hunter2Secr3t'], + ['{"yaml":"password:\\t\'Adm1nPassw0rd\'"}', '{"yaml":"[redacted]"}', 'Adm1nPassw0rd'], + ['{"yaml":"password:\\n \'Adm1nPassw0rd\'"}', '{"yaml":"[redacted]"}', 'Adm1nPassw0rd'], + // An opaque value runs to its delimiter, the backslash before a quote included. + [ + '{"m":"{\\"u\\":\\"https://h.example/cb?sig=AbCdEfGhIjKlMnOpQrStUvW\\"}"}', + '{"m":"{\\"u\\":\\"https://h.example/cb?sig=[redacted]"}"}', + 'AbCdEfGhIjKlMnOpQrStUvW', + ], + // A quote escaped one level further down stays inside the value. + [ + '{"cmd":"export TOKEN=\\"Qz\\\\\\"SecretTail\\" end"}', + '{"cmd":"export [redacted] end"}', + 'SecretTail', + ], + // An escaped backslash escapes only the character right after it. + [ + '{"cmd":"export TOKEN=\\"Qz7abcdef\\\\x\\" public"}', + '{"cmd":"export [redacted] public"}', + 'Qz7abcdef', + ], + ]); + }); + + it('reads a quoted value that does not close on its line as an unquoted run', () => { + const runs = [ + ['{"password":"hunter2hunter2', '{"[redacted]', 'hunter2hunter2'], + ['password: "hunter2hunter2\nnext line', '[redacted]\nnext line', 'hunter2hunter2'], + ["DB_PASSWORD='hunter2hunter2\nNEXT=1", '[redacted]\nNEXT=1', 'hunter2hunter2'], + ['api_key="Qz7abcdefghij\\', '[redacted]', 'Qz7abcdefghij'], + // The run ends at the delimiters of an unquoted value and at its line's end, which a CR, a + // backslash before one or a lone backslash also is. + ['password="Qz7abcdef public tail', '[redacted] public tail', 'Qz7abcdef'], + ['password="Qz7abcdef,next', '[redacted],next', 'Qz7abcdef'], + ['password="Qz7abcdef;next', '[redacted];next', 'Qz7abcdef'], + ['password="Qz7abcdef&x=1', '[redacted]&x=1', 'Qz7abcdef'], + ['password="Qz7abcdef\'x', "[redacted]'x", 'Qz7abcdef'], + ['password="Qz7abcdef\tpublic', '[redacted]\tpublic', 'Qz7abcdef'], + ['password="Qz7abc\ndef\nghi tail', '[redacted]\ndef\nghi tail', 'Qz7abc'], + ['password="Qz7abc\ndef\\', '[redacted]\ndef\\', 'Qz7abc'], + ['password="Qz7abc\rdef tail', '[redacted]\rdef tail', 'Qz7abc'], + ['password="Qz7abc\\\rdef tail', '[redacted]\rdef tail', 'Qz7abc'], + ['password="Qz7abcdef\\\nmore" tail', '[redacted]\nmore" tail', 'Qz7abcdef'], + ['password="Qz7abcdef\\\rmore" tail', '[redacted]\rmore" tail', 'Qz7abcdef'], + // A value that closes on a later line is read the same way, and a run of such values settles in + // one pass. + ['password="Qz7abcdef\nmore" tail', '[redacted]\nmore" tail', 'Qz7abcdef'], + [ + 'A_TOKEN="Qz7aaa1\nB_TOKEN=\'Qz7bbb2\nC_TOKEN="Qz7ccc3\nD_TOKEN="Qz7ddd4"', + '[redacted]\n[redacted]\n[redacted]\n[redacted]', + 'Qz7bbb2', + ], + // One JSON level down only an escaped quote closes the value; the raw quote or raw line break + // that ends the string around it, like an escaped line break, ends its run. + [ + '{"a":"password=\\"Qz7abcdefgh","b":"public"}', + '{"a":"[redacted]","b":"public"}', + 'Qz7abcdefgh', + ], + ['{"a":"password=\\"Qz7abcdef ghi","b":"x"}', '{"a":"[redacted] ghi","b":"x"}', 'Qz7abcdef'], + [ + '{"a":"password=\\"Qz7abcdefgh\\nline two","b":"public"}', + '{"a":"[redacted]\\nline two","b":"public"}', + 'Qz7abcdefgh', + ], + [ + '{"a":"password=\\"Qz7abcdefgh\\rline two","b":"public"}', + '{"a":"[redacted]\\rline two","b":"public"}', + 'Qz7abcdefgh', + ], + [ + '{"a":"password=\\"Qz7abc\\nQz7def\\" tail"}', + '{"a":"[redacted]\\nQz7def\\" tail"}', + 'Qz7abc', + ], + [ + '{"a":"password=\\"Qz7abcdefgh\n{"b":"say \\"hi\\""}', + '{"a":"[redacted]\n{"b":"say \\"hi\\""}', + 'Qz7abcdefgh', + ], + [ + 'echo password=\\"Qz7abc\\\nline two \\\nline three\nend', + 'echo [redacted]\nline two \\\nline three\nend', + 'Qz7abc', + ], + [ + '{"a":"A_TOKEN=\\"Qz7aaa1\\n B_TOKEN=\\"Qz7bbb2\\n C_TOKEN=\\"Qz7ccc3\\n D_TOKEN=\\"Qz7ddd4\\""}', + '{"a":"[redacted]\\n [redacted]\\n [redacted]\\n [redacted]"}', + 'Qz7aaa1', + ], + // A raw quote of the other kind does not end it: an apostrophe is text in a JSON string. + [ + '{"a":"token=\\\'Qz7secret","b":"public one"}', + '{"a":"[redacted]","b":"public one"}', + 'Qz7secret', + ], + // A value whose line break another mask of the pass takes (a scheme label's token, a nested + // key's value) closes as the next pass reads it, and is masked whole. + ['DB_PASSWORD="Bearer\nabcdefghij Zq7Wk3Xp9Rm2" tail', '[redacted] tail', 'Zq7Wk3Xp9Rm2'], + [ + 'Authorization: "Bearer\r\nabcdefghij Zq7Wk3Xp9Rm2" tail', + '[redacted] tail', + 'Zq7Wk3Xp9Rm2', + ], + ['password="abcdef Bearer\nabcdefghij Zq7Wk3Xp9Rm2" tail', '[redacted] tail', 'Zq7Wk3Xp9Rm2'], + ['token:"Kx96Wv Qze6S0YzVYHq Bearer\nQVI3ml-K"', '[redacted]', 'Qze6S0YzVYHq'], + [ + 'client_secret="token:\nabcdefghijkl Zq7Wk3Xp9Rm2Vn5Ty8Lb" end', + '[redacted] end', + 'Zq7Wk3Xp9Rm2Vn5Ty8Lb', + ], + // A quote that closes another value is that value's: no value it opens is read through the + // other masks. + ["token='abc password='xyz] public", '[redacted]xyz] public', 'abc'], + // So does one whose escaped closing quote an opaque value's mask unescapes, behind an odd or + // an even run of backslashes. + [ + 'password="say \\"Zq7Wk3Xp9Rm2Vn5Ty8Lb?sig=AAAAAAAAAAAAAAAAAAAAAAAA\\"', + '[redacted]', + 'Zq7Wk3Xp9Rm2Vn5Ty8Lb', + ], + [ + 'password="my Zq9SECRETvalue7Xk?sig=AbCdEfGhIjKlMnOpQrStUvWx\\\\" next', + '[redacted] next', + 'Zq9SECRETvalue7Xk', + ], + ] as const; + projects('SECRETS_B8_CORE_UNCLOSED_QUOTE', runs); + for (const [input, expected] of runs) { + expect(projectSecretShapedValue({ a: input }), 'SECRETS_B8_CORE_UNCLOSED_QUOTE').toEqual({ + value: { a: expected }, + settled: true, + }); + } + // Its quote may instead close a string the key sits in, so the text after the run stays, and calls + // that differ only there keep distinct projections. A run under six characters is no value. + const script = 'read -p "Enter password: " PW\necho "connecting to $HOST"\n'; + for (const kept of [ + script, + '{\n "label": "Password:",\n "hint": "at least 8 chars"\n}', + 'print("token: " + t)\nprint("done")', + 'fix(auth): stop logging "token=" query values in the billing service', + "grep -rn 'api_key=' config/prod.yaml", + `psql "password="$PW -c 'DROP TABLE orders_a'`, + '{"msg":"stop logging \\"token=\\" values in billing"}', + 'A_TOKEN="Qz7a1\nx=\\\\"y', + ]) { + expect(redactSecretShapedText(kept), 'SECRETS_B8_CORE_UNCLOSED_QUOTE').toBe(kept); + } + for (const [one, other] of [ + [script, script.replace('PW\n', 'PW; rm -rf ./build\n')], + [ + 'fix(auth): stop logging "token=" query values in the billing service', + 'fix(auth): stop logging "token=" query values in the payments service', + ], + [ + `psql "password="$PW -c 'DROP TABLE orders_a'`, + `psql "password="$PW -c 'DROP TABLE orders_b'`, + ], + [ + '{"msg":"stop logging \\"token=\\" values in billing"}', + '{"msg":"stop logging \\"token=\\" values in payments"}', + ], + [ + '{"cmd":"export TOKEN=\\"Qz7abcdef; echo done"}', + '{"cmd":"export TOKEN=\\"Qz7abcdef; rm -rf x"}', + ], + ]) { + expect( + projectSecretShapedValue({ content: one }).value, + 'SECRETS_B8_CORE_UNCLOSED_QUOTE', + ).not.toEqual(projectSecretShapedValue({ content: other }).value); + } + }); + + it('reads a label at a line start inside a split continuation', () => { + projects('SECRETS_B8_CORE_CONTINUATION_LABEL', [ + [ + 'sk-proj-AbCdEfGhIjKlMnOpQrSt\nauthori\tzation: Qz7SecretValue99', + '[redacted]', + 'Qz7SecretValue99', + ], + ['glpat-AbCdEfGhIjKlMnOpQrSt\napi-k\tey: Qz7SecretValue99', '[redacted]', 'Qz7SecretValue99'], + ]); + }); + + it('masks two and three private-key blocks in one text', () => { + const block = (kind: string, body: string) => + `-----BEGIN ${kind} PRIVATE KEY-----\n${body}\n-----END ${kind} PRIVATE KEY-----`; + projects('SECRETS_B8_CORE_PEM_BLOCKS', [ + [ + `a ${block('RSA', 'AAA')}\nb ${block('EC', 'BBBSECRET')} c`, + 'a [redacted]\nb [redacted] c', + 'BBBSECRET', + ], + [ + `${block('RSA', 'X1SECRET')} ${block('EC', 'X2SECRET')} ${block('OPENSSH', 'X3SECRET')}`, + '[redacted] [redacted] [redacted]', + 'X3SECRET', + ], + ]); + }); + + it("keeps a label's own whitespace before a split credential", () => { + expect( + redactSecretShapedText('Bearer sk-\nabcdefghijklmnopqrstuvwx done'), + 'SECRETS_B8_CORE_FILL', + ).toBe('Bearer [redacted] done'); + }); + it('summarizes a value that closes on a later line whole', async () => { + for (const [result, summary, secret] of [ + [ + 'token=\'abc password="first-half\nSECOND-HALF-SECRET" done', + '[redacted] done', + 'SECOND-HALF', + ], + ['password="password="password="\nQ7mZ4pW9xK"', '[redacted]', 'Q7mZ4pW9xK'], + ['secret: \'line1\nline2 token="it\'s"', '[redacted]', 'line2'], + [{ content: 'password="hunter2 value\nmore text"' }, '{"content":"[redacted]"}', 'hunter2'], + [ + { stdout: 'password="first-half\r\nSECOND-HALF" done' }, + '{"stdout":"[redacted] done"}', + 'SECOND-HALF', + ], + [['password="hunter2 value\nmore text"'], '["[redacted]"]', 'more text'], + // A masked value after it can take the quote that would close it. + ["token='\ntoken=''Qz7secret'", '[redacted]', 'Qz7secret'], + ['password="\npassword=""Qz7secret"', '[redacted]', 'Qz7secret'], + // When masked values take every later quote, the summary masks through the last of them. + ["token='\ntoken=''Qz7secret token='", '[redacted]', 'Qz7secret'], + // A structured result is read in its JSON, where the value one JSON string down runs on over the + // escaped line break to the end of the text. + [ + { stdout: 'token=\'abc password="first-half\nSECOND-HALF-SECRET" done' }, + '{"stdout":"[redacted]', + 'SECOND-HALF', + ], + ['{"log":"password=\\"Qz7secret\\nmore\\" ok"}', '{"log":"[redacted] ok"}', 'Qz7secret'], + ] as const) { + const bounded = await boundForModel(result, BIG, {}); + expect(bounded.summary, 'SECRETS_B8_CORE_SUMMARY_CLOSER').toBe(summary); + expect(bounded.summary.includes(secret), 'SECRETS_B8_CORE_SUMMARY_CLOSER').toBe(false); + } + // A value that reaches the end of the text unclosed is masked to that end. The nearest quote that + // closes the value ends its mask, so public text after it stays; a raw line break behind a backslash + // ends a value one JSON string down, and a quote behind three backslashes closes no value, one + // JSON string down too, where one behind five closes it. Each value is decided after its own line: + // a quote there that a nested value's mask leaves unescaped closes nothing. + for (const [result, summary] of [ + ['token="Qz7abcdef ghi jkl', '[redacted]'], + ['password="Qz7abc\ndefSECRET\\\\" tail', '[redacted] tail'], + ['password="Qz7abc\ndefSECRET\\\\\\\\" tail', '[redacted] tail'], + ['token="Qz7first\ntoken="Qz7second\nend', '[redacted] end'], + ['{"cmd":"export TOKEN=\\"Qz7abcdef; echo done"}', '{"cmd":"export [redacted]"}'], + ['token:"token=\\\'token=h\\\\\\r\\"\n"', '[redacted]'], + ['password=\\"Qz7abcdefgh\\nx \\\\\\" y', '[redacted]\\nx \\\\\\" y'], + ['password=\\"Qz7abcdefgh\\nx \\\\\\\\\\" y', '[redacted] y'], + ['password="Qz7abc\ndef" public "kept"', '[redacted] public "kept"'], + ['echo password=\\"Qz7abc\\\nline two \\" end', 'echo [redacted] line two \\" end'], + ['password="Qz7abcdefgh\nmore \\\\\\" text', '[redacted] more \\\\\\" text'], + ] as const) { + expect((await boundForModel(result, BIG, {})).summary, 'SECRETS_B8_CORE_SUMMARY_CLOSER').toBe( + summary, + ); + } + // The journal projection reads such a value only to its first line's end: its quote may close a + // string around it. + expect( + redactSecretShapedText('password="first-half\nSECOND-HALF" done'), + 'SECRETS_B8_CORE_SUMMARY_CLOSER', + ).toBe('[redacted]\nSECOND-HALF" done'); + }); + + it("reads a structured result's values across its line breaks and members", async () => { + // The text rules read the JSON as one text: a JSON-escaped line break is no delimiter, and a PEM + // block or a quoted value that runs on into later members keeps its key and markers. + const pem = [ + '-----BEGIN RSA PRIVATE KEY-----', + 'MIIEowIBAAKCAQEAuY5wq3Zx9bQk1s8hR2pV7tX0aLmN4cQe', + 'Kq1sW9eR4tY7uI0oP2aS5dF8gH1jK3lZ', + '-----END RSA PRIVATE KEY-----', + ]; + const signed = + 'curl -sS https://bucket.s3.example/obj?X-Amz-Signature=0123456789abcdef0123456789abcdef'; + for (const [result, summary, secret] of [ + [ + { + lines: [ + 'APP_NAME=demo', + `PRIVATE_KEY="${pem[0]}`, + pem[1], + pem[2], + `${pem[3]}"`, + 'PORT=8080', + ], + }, + '{"lines":["APP_NAME=demo","[redacted]","PORT=8080"]}', + 'MIIEowIBAAKCAQEA', + ], + [ + { lines: [`password=${pem[0]}`, pem[1], pem[3]] }, + '{"lines":["[redacted]"]}', + 'MIIEowIBAAKCAQEA', + ], + [ + { lines: ["export DB_PASSWORD='first", "hunter2secretpw'", 'echo ok'] }, + '{"lines":["export [redacted]","echo ok"]}', + 'hunter2secretpw', + ], + [{ stdout: "?token='\nhunter2secretpw\\'" }, '{"stdout":"?[redacted]"}', 'hunter2secretpw'], + ...['\n', '\r\n', '\r', '\t'].map( + (separator) => + [ + { stdout: `DB_PASSWORD=hunter2hunter2${separator}Q9zXw7TokenValue` }, + '{"stdout":"[redacted]"}', + 'Q9zXw7', + ] as const, + ), + ...['\n', '\r\n', '\t'].map( + (separator) => + [ + { stdout: `${signed}${separator}0123456789abcdefSECRETCONTINUATION -o out` }, + '{"stdout":"curl -sS https://bucket.s3.example/obj?X-Amz-Signature=[redacted] -o out"}', + 'SECRETCONTINUATION', + ] as const, + ), + [ + { + content: [ + { + type: 'text', + text: 'private_key=MIIEvQIBADANBgkqhkiG9w0BAQEFAASC\nSECONDLINEu8XJk2Wq4tRmYv0yZp3\nTHIRDLINEaBcDeFgHiJk', + }, + ], + }, + '{"content":[{"type":"text","text":"[redacted]"}]}', + 'SECONDLINE', + ], + ] as const) { + const bounded = await boundForModel(result, BIG, {}); + expect(bounded.summary, 'SECRETS_B8_CORE_SUMMARY_JSON').toBe(summary); + expect(bounded.summary.includes(secret), 'SECRETS_B8_CORE_SUMMARY_JSON').toBe(false); + } + }); + + it('shows a summary that does not settle as its own mask', async () => { + // Four glued ids need four passes, so neither summary pass settles. + const run = 'AKIAAAAAAAAAAAAAAAAAASIABBBBBBBBBBBBBBBB'.repeat(2); + for (const [result, summary] of [ + [run, '[redacted]'], + [{ stdout: run }, '[redacted]'], + ] as const) { + expect( + (await boundForModel(result, BIG, {})).summary, + 'SECRETS_B8_CORE_SUMMARY_UNSETTLED', + ).toBe(summary); + } + }); + + it( + 'summarizes a many-part result within the case deadline', + async () => { + // An MCP result near the 4 MiB http/sse message bound: every member is walked once, and its JSON + // is read as one text. + const parts = [{ type: 'text', text: 'token=Qz7manyPartSecret' }]; + for (let index = 0; index < 150_000; index += 1) { + parts.push({ type: 'text', text: `part ${index}` }); + } + const { summary } = await boundForModel({ content: parts }, BIG, {}); + expect(summary.includes('Qz7manyPartSecret'), 'SECRETS_B8_CORE_SUMMARY_PARTS').toBe(false); + expect( + summary.startsWith('{"content":[{"type":"text","text":"[redacted]"},'), + 'SECRETS_B8_CORE_SUMMARY_PARTS', + ).toBe(true); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); + + it('reports a journal projection whose member names collide as unsettled', () => { + const colliding = { + 'sk-AAAAAAAAAAAAAAAAAAAA': 'rm -rf build', + 'sk-BBBBBBBBBBBBBBBBBBBB': 'echo done', + }; + expect(projectSecretShapedValue(colliding), 'SECRETS_B8_CORE_KEY_COLLISION').toEqual({ + value: { '[redacted]': 'echo done' }, + settled: false, + }); + expect( + projectSecretShapedValue({ + nested: { 'https://u:pw1@h/x': 'one', 'https://u:pw2@h/x': 'two' }, + }).settled, + 'SECRETS_B8_CORE_KEY_COLLISION', + ).toBe(false); + // The report reaches objects inside arrays too. + for (const value of [{ list: [colliding] }, { argv: ['run', colliding] }]) { + expect(projectSecretShapedValue(value).settled, 'SECRETS_B8_CORE_KEY_COLLISION').toBe(false); + } + // Names that stay distinct settle, and the display projection still shows one member. + expect( + projectSecretShapedValue({ 'password=a': 1, 'password=b': 2 }).settled, + 'SECRETS_B8_CORE_KEY_COLLISION', + ).toBe(true); + expect(redactSecretShapedValue(colliding), 'SECRETS_B8_CORE_KEY_COLLISION').toEqual({ + '[redacted]': 'echo done', + }); + }); +}); + +describe('PR95 review round 8, sixth pre-review: escapes under nested masks, PEM markers, escaped separators', () => { + it('closes a quoted value at the later of its readings with and without the masks', async () => { + // A mask can take the backslash that escapes a later quote. A quoted value closes at the later of two + // readings: with the masks before and inside it filled, as the next pass reads them, and with each + // mask keeping the backslashes it ends with, as the text has them. A nested key's value that takes + // the backslash before a later quote therefore does not close the value early, in the journal + // projection, whose line break a scheme token takes, and in the summary alike; after an even run of + // backslashes the quote is unescaped in both readings and closes the value. + const a26 = 'a'.repeat(26); + for (const [input, expected] of [ + ['password="Bearer\nabcdefgh token=xyzxyz\\"secretpart" more', '[redacted] more'], + [ + 'password="a Bearer\nabcdefgh9 DB_PASSWORD=x9y8z7\\"Sup3rS3cretTail" next', + '[redacted] next', + ], + ['password="Bearer\nabcdefgh token=xyzxyz\\\\"tail more" end', '[redacted]tail more" end'], + ['password="Bearer\nabcdefgh token=xyzxyz\\\\\\"secret tail" end', '[redacted] end'], + // A value that closes only where a nested key's mask unescapes its last quote is masked whole. + ['"token":"Qz7abcdefghijkl,Qz7mnopqrstuv&\\ttoken:password:\\t\\"', '"[redacted]'], + // An opaque value's mask runs to the escaped quote and takes its backslash; a quote it unescapes + // closes the value only when no later quote does, behind a URL userinfo in it or not. + [`password="?sig=${a26}https://u:p@h\\"Qz7SecretTail" public`, '[redacted] public'], + [ + `password="token\nabcdefgh9 ?sig=${a26}https://u:p@h\\"Qz7SecretTail" public`, + '[redacted] public', + ], + [`password="x ?sig=${a26}\\"Qz7SecretTail" public`, '[redacted] public'], + // The earlier rules' masks stay filled in both readings: a quote inside a private-key block + // closes nothing. + [ + 'password="Bearer\nabcdefgh -----BEGIN RSA PRIVATE KEY----- x"y -----END RSA PRIVATE KEY----- tail" more', + '[redacted] more', + ], + ] as const) { + expect(redactSecretShapedText(input), 'SECRETS_B8_CORE_CLOSER_ESCAPE').toBe(expected); + expect(projectSecretShapedValue({ a: input }), 'SECRETS_B8_CORE_CLOSER_ESCAPE').toEqual({ + value: { a: expected }, + settled: true, + }); + } + // A summary masks a value that stops at a line break through the later of the two readings' closers. + for (const [input, expected] of [ + ['export PASSWORD="line1\nDB_TOKEN=abcdef\\"Secr3tTail" done', 'export [redacted] done'], + [ + 'export PASSWORD="line1\nDB_TOKEN=abcdef\\"Secr3tTail" done "kept"', + 'export [redacted] done "kept"', + ], + ['password="Qz7abc\nDB_TOKEN=abcdef\\"', '[redacted]'], + ['password="Qz7abc\nDB_TOKEN=abcdef\\"SECRETtail" done', '[redacted] done'], + ['password="Qz7abc\nDB_TOKEN=abcdef\\\\"tail" done', '[redacted]tail" done'], + ['password="Qz7abc\nDB_TOKEN=abcdef\\\\\\"SECRET tail" done', '[redacted] done'], + [`password="Qz7abc\nx ?sig=${a26}\\"Qz7SecretTail" public`, '[redacted] public'], + [`password="Qz7abc\nx ?sig=${a26}\\"`, '[redacted]'], + // The second reading falls back to the last quote that could close the value, like the first. + ['password="Qz7\ntoken="abcdef" token=zzzzzz\\" token="ghijkl" tail', '[redacted] tail'], + // Each value is decided at its own end, whatever order the values were found in. + ['passwd="\ntoken:"\n" tail', '[redacted] tail'], + ['passwd="token=\\\'\\rtoken=ef\\\\n\\"\ntoken=bcdef\\"', '[redacted]'], + ] as const) { + expect((await boundForModel(input, BIG, {})).summary, 'SECRETS_B8_CORE_CLOSER_ESCAPE').toBe( + expected, + ); + } + }); + + it('keeps the private-key markers a structured summary masks whole', async () => { + const begin = '-----BEGIN RSA PRIVATE KEY-----'; + const end = '-----END RSA PRIVATE KEY-----'; + // A credential member or flag argument masked whole keeps a marker whose partner lies outside it, + // so a block that runs on into or out of it is still masked across members. + for (const [result, summary, secret] of [ + [ + { private_key: `${begin}\nMIIEAAAAfirst`, rest: `MIIEBBBBsecondbodyline\n${end}` }, + '{"[redacted]}', + 'MIIEBBBB', + ], + [ + { a: `${begin}\nMIIEpAIBAAKCAQEAbodylineone`, private_key: `bodylinetwo\n${end}` }, + '{"a":"[redacted]"}', + 'bodylineone', + ], + [ + ['--private-key', begin, 'MIIEpAIBAAKCAQEAsecretbodyline1', 'abcdefsecretbodyline2', end], + '["--private-key","[redacted][redacted]"]', + 'secretbodyline', + ], + [ + [`--private-key=${begin}`, 'MIIEpAIBAAKCAQEAsecretbodyline1', end], + '["--[redacted]"]', + 'MIIE', + ], + // A member that ends one block and starts another keeps both markers. + [ + { + a: `${begin}\nAAAbody1`, + private_key: `body1tail\n${end}\n-----BEGIN EC PRIVATE KEY-----\nbody2head`, + c: 'BBBbody2\n-----END EC PRIVATE KEY-----', + d: 'public', + }, + '{"a":"[redacted]","d":"public"}', + 'BBBbody2', + ], + // The END marker it keeps is its first, the one that may close a block begun before it. + [ + { + a: `${begin}\nAAAfirstbody`, + private_key: `bbb\n${end}\n${begin}\nccc\n${end}`, + z: 'public', + }, + '{"a":"[redacted]","z":"public"}', + 'AAAfirstbody', + ], + // The BEGIN marker it keeps is its last, the one that may open a block that runs on after it. + [ + { private_key: `${begin}\nAAA\n${end}\n${begin}\nBBBhead`, rest: `CCCbody\n${end}` }, + '{"[redacted]}', + 'CCCbody', + ], + // Members inside arrays and objects keep them too. + [ + { items: [{ private_key: `${begin}\nMIIEnested`, rest: `NESTEDbody\n${end}` }] }, + '{"items":[{"[redacted]}]}', + 'NESTEDbody', + ], + ] as const) { + const bounded = await boundForModel(result, BIG, {}); + expect(bounded.summary, 'SECRETS_B8_CORE_SUMMARY_PEM').toBe(summary); + expect(bounded.summary.includes(secret), 'SECRETS_B8_CORE_SUMMARY_PEM').toBe(false); + } + // A block inside the member is masked with its markers, so a lone marker elsewhere stays text. + for (const [result, summary] of [ + [ + { a: `docs mention ${begin} here`, private_key: `${begin}\nMIIEwhole\n${end}` }, + `{"a":"docs mention ${begin} here","[redacted]}`, + ], + [ + { private_key: `${begin}\nMIIEwhole\n${end}`, b: `see ${end} there` }, + `{"[redacted],"b":"see ${end} there"}`, + ], + ] as const) { + expect((await boundForModel(result, BIG, {})).summary, 'SECRETS_B8_CORE_SUMMARY_PEM').toBe( + summary, + ); + } + // The journal and the display projections keep their whole mask. + expect( + projectSecretShapedValue({ private_key: `${begin}\nMIIEAAAA`, rest: 'x' }).value, + 'SECRETS_B8_CORE_SUMMARY_PEM', + ).toEqual({ private_key: '[redacted]', rest: 'x' }); + expect( + redactSecretShapedValue(['--private-key', begin]), + 'SECRETS_B8_CORE_SUMMARY_PEM', + ).toEqual(['--private-key', '[redacted]']); + }); + + it("reads a structured summary's JSON-escaped line breaks and tabs", async () => { + const jwt = + 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U'; + // JSON writes a line break or a tab in a member as two characters, which the summary reads as that + // separator for a credential or a scheme label after it and for a credential split by it. + for (const [result, summary, secret] of [ + [ + { stdout: 'keys:\nsk-proj-abcdefghijklmnopqrstuvwxyz', stderr: '', exitCode: 0 }, + '{"stdout":"keys:\\n[redacted]","stderr":"","exitCode":0}', + 'sk-proj-abc', + ], + [ + { stdout: 'ACCESSKEY\tAKIAIOSFODNN7EXAMPLE\tActive\n' }, + '{"stdout":"ACCESSKEY\\t[redacted]\\tActive\\n"}', + 'AKIAIOSFODNN7', + ], + [ + { stdout: 'one sk-proj-aaaaaaaaaaaaaaaaaaaaaaaa\r\nsk-proj-bbbbbbbbbbbbbbbbbbbbbbbb' }, + '{"stdout":"one [redacted]\\r\\n[redacted]"}', + 'bbbbbbbb', + ], + [{ stdout: 'AKIAABCDEFGH\nIJKLMNOP' }, '{"stdout":"[redacted]"}', 'IJKLMNOP'], + [ + { stdout: 'progress 50%\rsk-proj-abcdefghijklmnopqrstuvwxyz' }, + '{"stdout":"progress 50%\\r[redacted]"}', + 'sk-proj-abc', + ], + [ + { stdout: 'Bearer\nabcdefghijklmnop' }, + '{"stdout":"Bearer [redacted]"}', + 'abcdefghijklmnop', + ], + [ + { content: [{ type: 'text', text: `Token issued:\n${jwt}` }] }, + '{"content":[{"type":"text","text":"Token issued:\\n[redacted]"}]}', + 'eyJhbGciOiJIUzI1NiJ9', + ], + // The backslash before the letter may end a longer run: a backslash before a line break, or a JSON + // text inside a member. + [ + { stdout: 'x\\\nsk-proj-abcdefghijklmnopqrstuvwxyz' }, + '{"stdout":"x\\\\\\n[redacted]"}', + 'sk-proj-abc', + ], + [ + { stdout: '{"msg":"line1\\nsk-proj-abcdefghijklmnopqrstuvwxyz"}' }, + '{"stdout":"{\\"msg\\":\\"line1\\\\n[redacted]\\"}"}', + 'sk-proj-abc', + ], + ] as const) { + const bounded = await boundForModel(result, BIG, {}); + expect(bounded.summary, 'SECRETS_B8_CORE_SUMMARY_ESCAPES').toBe(summary); + expect(bounded.summary.includes(secret), 'SECRETS_B8_CORE_SUMMARY_ESCAPES').toBe(false); + } + // A path's backslashes before a letter add no mask without a credential after them. + expect( + (await boundForModel({ stdout: 'C:\\new\\token path\\nothing here' }, BIG, {})).summary, + 'SECRETS_B8_CORE_SUMMARY_ESCAPES', + ).toBe('{"stdout":"C:\\\\new\\\\token path\\\\nothing here"}'); + // The journal projection, which is compared rather than shown, reads no escape. + expect( + (await boundForModel({ msg: 'Use\nToken handling' }, BIG, {})).summary, + 'SECRETS_B8_CORE_SUMMARY_ESCAPES', + ).toBe('{"msg":"Use\\nToken [redacted]"}'); + expect( + redactSecretShapedText('{"msg":"Use\\nToken handling"}'), + 'SECRETS_B8_CORE_SUMMARY_ESCAPES', + ).toBe('{"msg":"Use\\nToken handling"}'); + }); +}); diff --git a/packages/core/src/tools/bounding.ts b/packages/core/src/tools/bounding.ts index c22c65ee..39dd2fc9 100644 --- a/packages/core/src/tools/bounding.ts +++ b/packages/core/src/tools/bounding.ts @@ -163,14 +163,15 @@ const JOINED_STANDALONE_CREDENTIAL = new RegExp(STANDALONE_CREDENTIAL.source.sli const SCHEME_CREDENTIAL = /\b(b[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|b[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c|t[\n\t]*o[\n\t]*k[\n\t]*e[\n\t]*n)\s+(?=[\w.~+/-]{8})[\w.~+/-]+={0,2}/gi; -// A secret-ish key (bounded wrappers keep this ReDoS-safe) + `=`/`:`. The optional `["']?` BEFORE the -// separator catches the JSON `"access_token":"…"` shape (an OAuth/egress response body). Kept an +// A secret-ish key (bounded wrappers keep this ReDoS-safe) + `=`/`:`. The optional quote BEFORE the +// separator catches the JSON `"access_token":"…"` shape (an OAuth/egress response body), and its +// optional backslash the same shape one JSON string down (`\"access_token\":\"…\"`). Kept an // analyzable LITERAL (not composed via `new RegExp`) so Sonar's static super-linear-runtime (S5852) // check still covers this security-critical pattern; its keyword-alternation breadth is deliberate // (and `apikey` is dropped — `api[_-]?key` already subsumes it). The value is read after it, in // `visitSecretKeyValues`. const SECRET_KEY = - /\b[\w-]{0,32}(?:password|passwd|secret|token|api[_-]?key|authorization|access[_-]?key|private[_-]?key|client[_-]?secret)[\w-]{0,16}["']?\s*[=:]\s*/gi; + /\b[\w-]{0,32}(?:password|passwd|secret|token|api[_-]?key|authorization|access[_-]?key|private[_-]?key|client[_-]?secret)[\w-]{0,16}(?:\\?["'])?\s*[=:]\s*/gi; /** * One recognized span of the projection's input and the text that replaces it. Every rule reports @@ -280,16 +281,6 @@ function visitJoinedStandaloneCredentials( if (match === null) continue; matchedUntil = JOINED_STANDALONE_CREDENTIAL.lastIndex; mask(start, matchedUntil, '[redacted]'); - // A credential glued to the end of this one starts one of its own, as after an applied mask, so a - // run of fixed-length ids (AKIA…ASIA…) is masked whole in one pass. - for (let glued = matchedUntil; /[A-Za-z]/.test(joined[glued] ?? ''); glued = matchedUntil) { - JOINED_STANDALONE_CREDENTIAL.lastIndex = 0; - if (!JOINED_STANDALONE_CREDENTIAL.test(preview(glued))) break; - JOINED_STANDALONE_CREDENTIAL.lastIndex = glued; - if (JOINED_STANDALONE_CREDENTIAL.exec(joined) === null) break; - matchedUntil = JOINED_STANDALONE_CREDENTIAL.lastIndex; - mask(glued, matchedUntil, '[redacted]'); - } } } @@ -356,18 +347,13 @@ function visitSplitSpans( } return target + removed; } - let previousEnd = -1; visit(joined, (start, end, replacement, url) => { // A separator right before a URL's scheme is outside the URL, so it splits nothing. const schemeStart = url === undefined ? undefined : advance(url.schemeStart, true); const originalStart = advance(start, !ownsEdges); const ownsUntil = url === undefined ? undefined : advance(url.ownsUntil, ownsEdges); const originalEnd = advance(end, ownsEdges); - // A span that starts where the visitor's previous one ended is glued to a credential, which bounds - // it as an applied mask would. - const glued = originalStart === previousEnd; - previousEnd = originalEnd; - if (!glued && boundary?.test(text[originalStart - 1] ?? '')) return; + if (boundary?.test(text[originalStart - 1] ?? '')) return; if (!/[\n\t]/.test(text.slice(schemeStart ?? originalStart, originalEnd))) return; // A split userinfo owns nothing past its first removed separator: the next line may hold a key // of its own (`https://\nDB_PASSWORD=P@ss`). @@ -413,24 +399,34 @@ function joinSplitHeaderLabels(text: string): string { // A label ends in ':' (visitJoinedHeaderLabels) and no fill adds one, so a text without one has none. if (!/[\n\t]/.test(text) || !text.includes(':')) return text; const masks: SecretRange[] = []; + const firstLines: SecretRange[] = []; const mask: SecretSpanVisitor = (start, end, replacement) => { masks.push({ start, end, replacement }); + firstLines.push({ start, end, replacement }); }; - visitSplitCredentials(text, mask); + // A split credential's LF/TAB continuation can take in a label at the start of the next line + // (`sk-…\napi-k\tey: `), so labels are read a second time with each split credential filled only + // up to its first removed separator. The credentials' own masks are unchanged. + visitSplitCredentials(text, (start, end, replacement) => { + masks.push({ start, end, replacement }); + const split = text.slice(start, end).search(/[\n\t]/); + firstLines.push({ start, end: split < 0 ? end : start + split, replacement }); + }); visitPemBlocks(text, mask); visitJoinedStandaloneCredentials(text, text, mask); - const boundaries = afterMasks(text, masks); const labels: SecretRange[] = []; - visitSplitSpans( - boundaries, - (joined, label) => { - visitJoinedHeaderLabels(boundaries, joined, label); - }, - (start, end, replacement) => { - labels.push({ start, end, replacement }); - }, - /[A-Za-z0-9_-]/, - ); + for (const boundaries of new Set([afterMasks(text, masks), afterMasks(text, firstLines)])) { + visitSplitSpans( + boundaries, + (joined, label) => { + visitJoinedHeaderLabels(boundaries, joined, label); + }, + (start, end, replacement) => { + labels.push({ start, end, replacement }); + }, + /[A-Za-z0-9_-]/, + ); + } return replaceRanges(text, labels); } @@ -440,6 +436,7 @@ function joinSplitHeaderLabels(text: string): string { * it read `[redacted] [redacted]`. */ function replaceRanges(text: string, ranges: readonly SecretRange[]): string { + if (ranges.length === 0) return text; const ordered = [...ranges].sort((left, right) => left.start - right.start); const parts: string[] = []; let copied = 0; @@ -466,6 +463,7 @@ function replaceRanges(text: string, ranges: readonly SecretRange[]): string { * end. Offsets stay valid, so a rule that reads it reports ranges of the original text. */ function fillRanges(text: string, ranges: readonly SecretRange[]): string { + if (ranges.length === 0) return text; const runs: { start: number; end: number }[] = []; for (const range of [...ranges].sort((left, right) => left.start - right.start)) { if (range.end <= range.start) continue; @@ -503,13 +501,27 @@ export function redactSecretShapedText(text: string): string { return toFixedPoint(redactSecretShapedOnce, text, '[redacted]'); } -function redactSecretShapedOnce(text: string): string { +/** + * The summary's projection (`makeSummary`). A summary is shown, never compared, so it differs in one decision + * (`visitSecretKeyValues`): a quoted value that stops at a line break is masked through a later quote that + * could close it, else to the end of its line, and one that reaches the end of its string or of the text + * without closing is masked to that end. Left to the one-line pass, such a value lost its key to a mask + * before it (`token='a password="first\nsecond" done`), and JSON escaping hid it in a structured result. + */ +function redactSecretShapedSummaryText(text: string): string { + return toFixedPoint((value) => redactSecretShapedOnce(value, true), text, '[redacted]'); +} + +function redactSecretShapedOnce(text: string, throughCloser = false): string { const source = joinSplitHeaderLabels(text); - return replaceRanges(source, secretRanges(source)); + return replaceRanges(source, secretRanges(source, throughCloser)); } -/** Every span the projection masks in `text`; each rule is described at its visitor. */ -function secretRanges(text: string): readonly SecretRange[] { +/** + * Every span the projection masks in `text`; each rule is described at its visitor. `throughCloser` selects + * the summary's decisions (`redactSecretShapedSummaryText`). + */ +function secretRanges(text: string, throughCloser: boolean): readonly SecretRange[] { const ranges: SecretRange[] = []; // Ranges merge instead of erasing context, yet some reads see the text as already masked: a key or // parameter in a URL userinfo's own text (`UrlUserinfo.ownsUntil`) is that userinfo's, a key's @@ -533,7 +545,11 @@ function secretRanges(text: string): readonly SecretRange[] { visitPemBlocks(text, report); // No value runs through a scheme label's token: `authorization: Bearer ` keeps both masks. visitSchemeCredentials(text, record); - visitSecretKeyValues(names, fillRanges(text, masked), record); + const values = fillRanges(text, masked); + // The same masks with the backslashes each ends with kept, for a quote's second reading (`laterClosing`). + const kept = keepingEscapes(text, masked); + const escapes = kept === masked ? values : fillRanges(text, kept); + const unclosed = visitSecretKeyValues(names, values, escapes, record, throughCloser); // Well-known standalone credential shapes (OpenAI, Stripe, AWS incl. STS ASIA/ABIA, GitHub token + PAT, // GitLab PAT, Slack, Google API + OAuth, HuggingFace, npm, JWT). Kept as ONE alternation ON PURPOSE: a // single leftmost-longest scan covers two ADJACENT / EMBEDDED credential shapes (e.g. a JWT whose @@ -550,19 +566,100 @@ function secretRanges(text: string): readonly SecretRange[] { }); // A scheme label glued to the end of another credential starts one of its own, even where a key's // value runs on over it: labels are read again with every mask filled but key values and scheme - // tokens (`afterMasks`). Anything else glued to a mask is read by the projection's next pass. - visitSchemeCredentials(afterMasks(text, [...masked, ...standalone]), record); + // tokens (`afterMasks`). Anything else glued to a mask is read by the projection's next pass. With no + // mask, that view is the text the labels were already read in. + if (masked.length > 0 || standalone.length > 0) { + visitSchemeCredentials(afterMasks(text, [...masked, ...standalone]), record); + } + // A key's quoted value that does not close on its line is decided last, when every other range of the + // pass is known (`visitUnclosedValues`). + if (unclosed.length > 0) visitUnclosedValues(text, values, ranges, unclosed, record); + // A summary also reads the line breaks and tabs that JSON escaping hides (`visitEscapedSeparators`). + if (throughCloser) visitEscapedSeparators(text, record); return ranges; } +/** + * The ranges without the run of backslashes each ends with in `text`, or `ranges` itself when none ends + * with one. Filled, they leave a quote right after a mask the escape it has in the text (`laterClosing`). + */ +function keepingEscapes(text: string, ranges: readonly SecretRange[]): readonly SecretRange[] { + let changed = false; + const kept = ranges.map((range) => { + let end = range.end; + while (end > range.start && text[end - 1] === '\\') end -= 1; + if (end === range.end) return range; + changed = true; + return { ...range, end }; + }); + return changed ? kept : ranges; +} + +/** Where a quoted value ends (`quotedValueEnd`). */ +type QuotedValueEnd = ReturnType; + +/** + * Of a quoted value's two readings, the one that closes it later: one where the masks before or inside it + * are filled, as the next pass reads them, and one where each mask keeps the backslashes it ends with, as + * the text has them. A mask can take the backslash that escapes a later quote, which then closes the value + * early in the first reading (`password="Bearer\nabcdefgh token=xyz\"tail" more`), or the backslash before + * a quote that is the value's only closer (`password="say \"…?sig=<24 or more>\"`), which closes it in the + * first reading alone; the later closing of the two masks both. A reading that closes beats one that does + * not. + */ +function laterClosing(first: QuotedValueEnd, second: QuotedValueEnd): QuotedValueEnd { + return second.closed && (!first.closed || second.end > first.end) ? second : first; +} + +/** + * A summary's second reading, for the line breaks and tabs that JSON escaping hides: a structured result's + * JSON writes each as two characters, so a credential or a scheme label right after one has no boundary, + * and a credential split by one is not joined. The rules that a separator decides read a view of the + * same length in which each such escape is its separator twice (`escapedSeparatorView`); their ranges + * merge with those of the JSON reading, which reads a quoted value or a PEM block across the escapes. + */ +function visitEscapedSeparators(text: string, report: SecretSpanVisitor): void { + const view = escapedSeparatorView(text); + if (view === text) return; + visitSplitCredentials(view, report); + visitJoinedStandaloneCredentials(view, view, report); + visitSchemeCredentials(view, report); +} + +/** + * `text` with each escaped line feed, carriage return or tab (`\n`, `\r`, `\t`) replaced by that separator + * twice, so offsets stay valid. The backslash before the letter may end a longer run, as in a JSON text + * inside a member (`\\n`) or a path (`C:\new`): the view only adds masks, and a mask needs a credential + * after the separator. Each run of backslashes is read once. + */ +function escapedSeparatorView(text: string): string { + const parts: string[] = []; + let copied = 0; + for (let index = text.indexOf('\\'); index >= 0; index = text.indexOf('\\', index)) { + let end = index; + while (text[end] === '\\') end += 1; + const letter = text[end]; + const separator = + letter === 'n' ? '\n' : letter === 'r' ? '\r' : letter === 't' ? '\t' : undefined; + if (separator !== undefined) { + parts.push(text.slice(copied, end - 1), separator, separator); + copied = end + 1; + } + index = end; + } + if (parts.length === 0) return text; + parts.push(text.slice(copied)); + return parts.join(''); +} + /** * Signed-URL/webhook values can be opaque credentials under an unrecognised query key. * Preserve short values, parameter names and public delimiters/fragments. Once recognized, * mask the whole value: a fixed-length prefix leaves a permanent credential equality oracle. * Scan delimiters separately instead of matching a multi-megabyte repeated value in V8's * RegExp stack. Both cursors advance; short unrecognized values add only bounded overlap. - * Parameters are read in `names`, which hides each URL userinfo's own text (a parameter there is the - * userinfo's); a value is measured in `text`, so a mask inside it neither ends nor extends it. + * Parameters are read in `names`, which hides each URL userinfo's username (`UrlUserinfo.ownsUntil`); + * a value is measured in `text`, so a mask inside it neither ends nor extends it. */ function visitOpaqueQueryValues(names: string, text: string, mask: SecretSpanVisitor): void { if (!names.includes('?') && !names.includes('&')) return; @@ -622,6 +719,8 @@ function visitPemBlocks(text: string, mask: SecretSpanVisitor): void { * equivalent `(?=class{N})class+` form keeps the whole value instead of clipping a credential. */ function visitSchemeCredentials(text: string, mask: SecretSpanVisitor): void { + // Whitespace follows every label, so a text without any has no scheme credential. + if (!/\s/.test(text)) return; const scheme = new RegExp(SCHEME_CREDENTIAL); for (let match = scheme.exec(text); match !== null; match = scheme.exec(text)) { const labelEnd = match.index + (match[1]?.length ?? 0); @@ -631,56 +730,264 @@ function visitSchemeCredentials(text: string, mask: SecretSpanVisitor): void { } /** - * Where the quoted value that opens at `start` ends, or -1 when it does not close: just past the - * first UNESCAPED matching quote, including JSON's escaped quotes/backslashes (otherwise a valid - * password such as `a\"secret-tail` left its tail exposed), and never across a CR or LF. A backslash - * consumes exactly the next character. A plain scan, because a RegExp repeat over the value keeps - * backtracking state per character and exhausts V8's stack on an admitted-size value. + * Where the quoted value that opens at `start` ends: just past its closing quote on its line (`closed`), + * at a `lineBreak` it does not cross (it may still close on a later line), or, without closing, at the + * end of the string around it or of the text. A value opened by a plain quote (`"…"`, `'…'`) closes at + * the first UNESCAPED matching quote: a backslash consumes exactly the next character, as in JSON's + * escaped quotes and backslashes (otherwise `a\"secret-tail` left its tail exposed). A value opened by an + * escaped quote (`\"…\"`, as JSON inside a JSON string writes it) is read one decoding level down: each + * backslash pair is one character, an escaped backslash escapes the next one, an escaped `\n`/`\r` is a + * line break, and only its escaped quote closes it. The raw quote of its kind or the raw line break, + * behind a backslash or not, ends the string around it, which the value cannot cross. A raw quote of the + * other kind does not end it: an apostrophe is ordinary text in a JSON string. A plain scan, because a + * RegExp repeat over the value keeps backtracking state per character and exhausts V8's stack on an + * admitted-size value. */ -function quotedValueEnd(text: string, start: number): number { - const quote = text[start]; - for (let index = start + 1; index < text.length; index += 1) { +function quotedValueEnd( + text: string, + start: number, +): { readonly end: number; readonly closed: boolean; readonly lineBreak: boolean } { + const escaped = text[start] === '\\'; + const quote = text[escaped ? start + 1 : start]; + // A raw line break stops a plain value; one JSON string down it ends the string around the value. + const stop = (end: number): { end: number; closed: boolean; lineBreak: boolean } => ({ + end, + closed: false, + lineBreak: !escaped, + }); + let escapeNext = false; + for (let index = escaped ? start + 2 : start + 1; index < text.length; index += 1) { const char = text[index]; - if (char === quote) return index + 1; - if (char === '\r' || char === '\n') return -1; - if (char === '\\') { - const next = text[index + 1]; - if (next === undefined || next === '\r' || next === '\n') return -1; - index += 1; + if (char === '\r' || char === '\n') return stop(index); + if (char === quote) + return escaped ? stop(index) : { end: index + 1, closed: true, lineBreak: false }; + if (char !== '\\') { + escapeNext = false; + continue; } + const next = text[index + 1]; + if (next === undefined) break; + if (next === '\n' || next === '\r') return stop(index + 1); + if (escaped && (next === 'n' || next === 'r')) { + return { end: index, closed: false, lineBreak: true }; + } + index += 1; + if (!escaped) continue; + if (escapeNext) escapeNext = false; + else if (next === quote) return { end: index + 1, closed: true, lineBreak: false }; + else escapeNext = next === '\\'; } - return -1; + return { end: text.length, closed: false, lineBreak: false }; +} + +/** Where an unquoted value, and the run of a quoted one that does not close, ends: whitespace, a quote, `,`, `;`, `&`. */ +function valueDelimiter(): RegExp { + return /[\s"',;&]/g; } /** - * A secret-ish key (`SECRET_KEY`) and its value: a whole quoted string, or an unquoted run of at least - * six characters up to whitespace, a quote, `,`, `;` or `&`. Keys are read in `keys`, which hides each - * URL userinfo's username (`UrlUserinfo.ownsUntil`): a key there names the user + * A secret-ish key (`SECRET_KEY`) and its value: a whole quoted string (`quotedValueEnd`, also one JSON + * string down), or an unquoted run of at least six characters up to whitespace, a quote, `,`, `;` or + * `&`. A quoted value that does not close on its line, or one JSON string down before the end of the + * string around it, is returned for the journal projection to decide once the pass's other ranges are + * known (`visitUnclosedValues`). The summary's projection (`throughCloser`) masks such a value through a + * later quote instead (`throughNearestCloser`). Keys are read in `keys`, which hides each URL + * userinfo's username (`UrlUserinfo.ownsUntil`): a key there names the user * (`https://x-access-token:@github.com/o/r`), and its value would run on through the public host and - * path. Values are read in `values`, where every earlier mask is one run, as an applied mask would be. - * The key scan resumes at each value rather than after it, so a key inside a value is read too: - * `token=a>password: ` has a second value past the first one's end. One delimiter scan serves - * every unquoted value, and a nested quoted value can only open at the closing quote of the one around - * it, so the scan stays linear. + * path. Values are read in `values`, where every earlier mask is one run, as an applied mask would be, + * and a quoted value also in `escapes`, where each such mask keeps the backslashes it ends with; it closes + * at the later of the two (`laterClosing`). The key scan resumes at each value rather than after it, so a + * key inside a value is read too: `token=a>password: ` has a second value past the first one's + * end. One delimiter scan serves every unquoted value, and a quoted value is scanned no further than its + * line: a nested quoted value can only open at the closing quote of the one around it, and one that does + * not close leaves no quote of its kind on its line, so the scan stays linear. */ -function visitSecretKeyValues(keys: string, values: string, mask: SecretSpanVisitor): void { +function visitSecretKeyValues( + keys: string, + values: string, + escapes: string, + report: SecretSpanVisitor, + throughCloser: boolean, +): readonly UnclosedValue[] { + // Every key ends in `=` or `:`, so names without either have no value to read. + if (!keys.includes('=') && !keys.includes(':')) return []; const key = new RegExp(SECRET_KEY); - const delimiter = /[\s"',;&]/g; + const delimiter = valueDelimiter(); let delimiterAt = -1; + const masked: SecretRange[] = []; + const open: OpenValue[] = []; + const unclosed: UnclosedValue[] = []; + const mask = (start: number, end: number): void => { + masked.push({ start, end, replacement: '[redacted]' }); + report(start, end, '[redacted]'); + }; for (let match = key.exec(keys); match !== null; match = key.exec(keys)) { - const valueStart = key.lastIndex; - const quote = values[valueStart]; - if (quote === '"' || quote === "'") { - const end = quotedValueEnd(values, valueStart); - if (end >= 0) mask(match.index, end, '[redacted]'); + const valueStart = escapedValueStart(values, key.lastIndex); + const escaped = values[valueStart] === '\\'; + const opening = escaped ? values[valueStart + 1] : values[valueStart]; + if (opening === '"' || opening === "'") { + const value = quotedValueEnd(values, valueStart); + const read = + escapes === values ? value : laterClosing(value, quotedValueEnd(escapes, valueStart)); + const kind = escaped ? `\\${opening}` : opening; + if (read.closed) mask(match.index, read.end); + else if (!throughCloser) + unclosed.push({ start: match.index, quote: valueStart, end: read.end }); + else if (read.lineBreak) open.push({ start: match.index, end: read.end, quote: kind }); + else mask(match.index, read.end); continue; } if (delimiterAt < valueStart) { delimiter.lastIndex = valueStart; delimiterAt = delimiter.exec(values)?.index ?? values.length; } - if (delimiterAt - valueStart >= 6) mask(match.index, delimiterAt, '[redacted]'); + if (delimiterAt - valueStart >= 6) mask(match.index, delimiterAt); + } + for (const range of throughNearestCloser(values, escapes, masked, open)) { + report(range.start, range.end, '[redacted]'); + } + return unclosed; +} + +/** A key's quoted value that does not close on its line: from its key, where its quote opens, where it stops. */ +interface UnclosedValue { + readonly start: number; + readonly quote: number; + readonly end: number; +} + +/** + * The journal projection's masks for key values that do not close on their line (`visitSecretKeyValues`). + * Each is read again in `closing`, where every other range of the pass is filled: a mask that takes its + * line break (a scheme label's token, a nested key's value) joins its lines as the next pass would read + * them, so a value that then closes is masked whole, as it was before this rule; it closes at the later of + * that reading and the one where each range keeps the backslashes it ends with (`laterClosing`). + * Otherwise it is read as an unquoted run from after its quote, in `values`, no further than where it + * stopped: its quote may instead close a string the key sits in (`"token=" in prose`), and masking on + * past the run would give calls that differ only in that later text one projection. A run starts right + * after a quote, where the delimiter scan of every earlier run stops, so no two runs share a scan; a + * reading in `closing` stops at the next quote that could close it, so the readings of one kind of quote + * share none either. + */ +function visitUnclosedValues( + text: string, + values: string, + ranges: readonly SecretRange[], + unclosed: readonly UnclosedValue[], + report: SecretSpanVisitor, +): void { + const closing = fillRanges(text, ranges); + const kept = keepingEscapes(text, ranges); + const escapes = kept === ranges ? closing : fillRanges(text, kept); + const delimiter = valueDelimiter(); + for (const value of unclosed) { + const from = value.quote + (values[value.quote] === '\\' ? 2 : 1); + // The quote itself may be inside another range, which then decides it. + if (closing.slice(value.quote, from) === values.slice(value.quote, from)) { + const first = quotedValueEnd(closing, value.quote); + const reread = + escapes === closing ? first : laterClosing(first, quotedValueEnd(escapes, value.quote)); + if (reread.closed) { + report(value.start, reread.end, '[redacted]'); + continue; + } + } + delimiter.lastIndex = from; + const end = Math.min(delimiter.exec(values)?.index ?? values.length, value.end); + if (end - from >= 6) report(value.start, end, '[redacted]'); + } +} + +/** + * Where a key's value starts: right after its separator, or one JSON string down behind escaped tabs and + * line breaks (`password:\t\"…\"`, `password:\n '…'`) when they end at an escaped quote or at an + * apostrophe, which JSON leaves unescaped; that quote then opens the value. A plain scan over the run, + * like `quotedValueEnd`'s. + */ +function escapedValueStart(text: string, start: number): number { + let index = start; + for (;;) { + const char = text[index]; + const next = text[index + 1]; + if (char === ' ' || char === '\t') index += 1; + else if (char === '\\' && (next === 't' || next === 'n' || next === 'r')) index += 2; + else break; + } + const char = text[index]; + const opening = text[index + 1]; + const quoted = char === "'" || (char === '\\' && (opening === '"' || opening === "'")); + return index > start && quoted ? index : start; +} + +/** A key's quoted value that stops at a line break, from its key; its quote is `\"` one JSON string down. */ +interface OpenValue { + readonly start: number; + readonly end: number; + readonly quote: string; +} + +/** + * The kind of open value the quote at `index` could close, as `quotedValueEnd` decodes it: after an even + * run of backslashes the quote is unescaped and may close a plain value; after a run of 4k+1 it closes a + * value one JSON string down. Each run is read once, before the quote it escapes. + */ +function closingKind(text: string, index: number): string | undefined { + const char = text[index]; + if (char !== '"' && char !== "'") return undefined; + let backslashes = 0; + while (text[index - 1 - backslashes] === '\\') backslashes += 1; + if (backslashes % 2 === 0) return char; + return backslashes % 4 === 1 ? `\\${char}` : undefined; +} + +/** + * The summary's masks for the `open` values (`redactSecretShapedSummaryText`). A summary is shown, never + * compared, so each value is masked through the nearest later quote that could close it outside the + * `masked` values, or, when every such quote is inside one, through the last of those, in its string or + * not, and only else to the end of its line. Masked values that take every later quote leave what lies + * between them uncertain (`token='\ntoken='''`), so the summary masks through all of it. Like a + * value that closes, it is read twice, with every mask filled and with each mask keeping the backslashes + * it ends with (`escapes`), and masked through the later of the two (`laterClosing`). One backward scan, + * from the end of the text to the earliest end of an open value; each run of backslashes is read once, + * before the quote it escapes. + */ +function throughNearestCloser( + values: string, + escapes: string, + masked: readonly SecretRange[], + open: readonly OpenValue[], +): readonly { readonly start: number; readonly end: number }[] { + if (open.length === 0) return []; + const byEnd = open.toSorted((left, right) => right.end - left.end); + const filled = fillRanges(values, masked); + const kept = keepingEscapes(escapes, masked); + const keptFilled = kept === masked && escapes === values ? filled : fillRanges(escapes, kept); + // Where the nearest quote of each kind outside the masked values ends, and where the last one does, in + // each reading. + const outside = new Map(); + const last = new Map(); + const keptOutside = new Map(); + const keptLast = new Map(); + const ranges: { start: number; end: number }[] = []; + let next = 0; + for (let index = values.length - 1; next < byEnd.length; index -= 1) { + for (let value = byEnd[next]; value !== undefined && value.end > index; value = byEnd[next]) { + next += 1; + const end = outside.get(value.quote) ?? last.get(value.quote) ?? value.end; + const keptEnd = keptOutside.get(value.quote) ?? keptLast.get(value.quote) ?? value.end; + ranges.push({ start: value.start, end: Math.max(end, keptEnd) }); + } + const kind = closingKind(filled, index); + if (kind !== undefined) outside.set(kind, index + 1); + const raw = closingKind(values, index); + if (raw !== undefined && !last.has(raw)) last.set(raw, index + 1); + if (keptFilled === filled) continue; + const keptKind = closingKind(keptFilled, index); + if (keptKind !== undefined) keptOutside.set(keptKind, index + 1); + const keptRaw = closingKind(escapes, index); + if (keptRaw !== undefined && !keptLast.has(keptRaw)) keptLast.set(keptRaw, index + 1); } + return ranges; } /** A host-shaped prefix: a dotted name (or IPv4), `localhost` or a bracketed IP literal, with an optional port. */ @@ -763,9 +1070,10 @@ export function redactSecretShapedValue(value: unknown): unknown { } /** - * The journal projection of a tool's arguments (`redactSecretShapedValue`) and whether every string in it - * settled. A string whose projection still changes on its third pass becomes `[redacted]`, which would give - * different calls one journal identity, so the registry refuses such an effect before it is prepared + * The journal projection of a tool's arguments (`redactSecretShapedValue`) and whether it settled. A string + * whose projection still changes on its third pass becomes `[redacted]`, and two member names of one object + * that project alike leave one member, either of which would give different calls one journal identity, so + * the registry refuses such an effect before it is prepared * ([effect-journal.md](../../../../docs/reference/shared-core/effect-journal.md) §11). */ export function projectSecretShapedValue(value: unknown): { @@ -773,18 +1081,64 @@ export function projectSecretShapedValue(value: unknown): { readonly settled: boolean; } { let settled = true; - const projected = redactSecretShapedWalk(value, new WeakSet(), (text) => { - const fixed = fixedPointOf(redactSecretShapedOnce, text); - if (fixed !== undefined) return fixed; - settled = false; - return '[redacted]'; - }); + const projected = redactSecretShapedWalk( + value, + new WeakSet(), + (text) => { + const fixed = fixedPointOf(redactSecretShapedOnce, text); + if (fixed !== undefined) return fixed; + settled = false; + return '[redacted]'; + }, + () => { + settled = false; + }, + ); return { value: projected, settled }; } /** How the walk projects each string: the display projection, or the journal's settle-checking one. */ type TextProjection = (text: string) => string; +/** What the walk puts in place of a value it masks whole: a credential member or a credential flag's argument. */ +type WholeMask = (value: unknown) => string; + +/** The journal's and the display's whole mask. */ +const maskWhole: WholeMask = () => '[redacted]'; + +/** + * The summary walk's whole mask (`toText`). It keeps a private-key END marker that may close a block opened + * before the value and a BEGIN marker that may open one continuing after it, so the summary's text rules + * still pair each with a marker in another member and mask the key's body there. A marker names a key + * type, not key material; a block whose markers are both inside the value is masked with it. + */ +function maskWholeKeepingPemMarkers(value: unknown): string { + if (typeof value !== 'string' || !value.includes('-----')) return '[redacted]'; + const begin = /-----BEGIN [A-Z0-9 ]{0,40}PRIVATE KEY-----/g; + const end = /-----END [A-Z0-9 ]{0,40}PRIVATE KEY-----/g; + let firstBegin: RegExpExecArray | undefined; + let lastBegin: RegExpExecArray | undefined; + for (let marker = begin.exec(value); marker !== null; marker = begin.exec(value)) { + firstBegin ??= marker; + lastBegin = marker; + } + let firstEnd: RegExpExecArray | undefined; + let lastEnd: RegExpExecArray | undefined; + for (let marker = end.exec(value); marker !== null; marker = end.exec(value)) { + firstEnd ??= marker; + lastEnd = marker; + } + const closing = + firstEnd !== undefined && (firstBegin === undefined || firstBegin.index > firstEnd.index) + ? firstEnd[0] + : ''; + const opening = + lastBegin !== undefined && (lastEnd === undefined || lastEnd.index < lastBegin.index) + ? lastBegin[0] + : ''; + return `${closing}[redacted]${opening}`; +} + /** * Does this OBJECT KEY name a secret? The same keyword alternation the `key=value` text rule uses, anchored * as a whole-name test rather than an inline one. @@ -870,12 +1224,14 @@ function redactArgVector( items: readonly unknown[], seen: WeakSet, project: TextProjection, + collide: (() => void) | undefined, + whole: WholeMask, ): unknown[] { const out: unknown[] = []; let redactNext: string | undefined; for (const item of items) { if (redactNext !== undefined) { - out.push(redactCredentialArgument(redactNext, item)); + out.push(redactCredentialArgument(redactNext, item, whole)); redactNext = undefined; continue; } @@ -884,22 +1240,22 @@ function redactArgVector( const flag = separator > 0 ? item.slice(0, separator) : ''; if (isSecretishFlag(flag)) { const value = item.slice(separator + 1); - out.push(`${flag}=${redactCredentialArgument(flag, value)}`); + out.push(`${flag}=${redactCredentialArgument(flag, value, whole)}`); continue; } if (isSecretishFlag(item)) redactNext = item; } - out.push(redactSecretShapedWalk(item, seen, project)); + out.push(redactSecretShapedWalk(item, seen, project, collide, whole)); } return out; } /** One credential-value rule for both argv spellings; only ambiguous -p may name a public port. */ -function redactCredentialArgument(flag: string, value: unknown): string { +function redactCredentialArgument(flag: string, value: unknown, whole: WholeMask): string { if (flag.toLowerCase() === '-p' && typeof value === 'string' && looksLikePort(value)) { return value; } - return '[redacted]'; + return whole(value); } /** @@ -944,9 +1300,11 @@ function redactUnderKey( item: unknown, seen: WeakSet, project: TextProjection, + collide: (() => void) | undefined, + whole: WholeMask, ): unknown { if (!isSecretishKey(key) || item === null || item === undefined) - return redactSecretShapedWalk(item, seen, project); + return redactSecretShapedWalk(item, seen, project, collide, whole); if (typeof item === 'string') { // Match only the bounded label and the start of a nonempty remainder, never return that remainder. // Canonical labels also keep caller-controlled casing out of the credential projection. @@ -960,20 +1318,23 @@ function redactUnderKey( return 'Token [redacted]'; } } - return '[redacted]'; + return whole(item); } function redactSecretShapedWalk( value: unknown, seen: WeakSet, project: TextProjection, + // Reports two member names of one object that project alike, which leaves only the later member. + collide?: () => void, + whole: WholeMask = maskWhole, ): unknown { if (typeof value === 'string') return project(value); if (typeof value !== 'object' || value === null) return value; if (seen.has(value)) return '[cyclic]'; // break the cycle (mirrors redactInlineMedia) — never re-emit the ref seen.add(value); try { - if (Array.isArray(value)) return redactArgVector(value, seen, project); + if (Array.isArray(value)) return redactArgVector(value, seen, project, collide, whole); if (!isPlainObject(value)) return value; // Date/RegExp/Map/… — leave for native handling const out: Record = {}; // Scrub the KEY too (a secret-shaped key is redacted; a normal name is unchanged — see the doc above). A @@ -983,8 +1344,10 @@ function redactSecretShapedWalk( // inspects strings by shape — so a numeric PIN, a nested credential object, or a high-entropy opaque // token with no recognisable prefix all survive it. `null`/`undefined` are left alone so an absent // field does not become a phantom `[redacted]` in a diagnostic. - Object.defineProperty(out, project(key), { - value: redactUnderKey(key, item, seen, project), + const name = project(key); + if (Object.hasOwn(out, name)) collide?.(); + Object.defineProperty(out, name, { + value: redactUnderKey(key, item, seen, project, collide, whole), enumerable: true, configurable: true, writable: true, @@ -996,34 +1359,66 @@ function redactSecretShapedWalk( } } -/** Render any result to the text the model would see (a string is itself; else compact JSON). Inline - * media bytes are redacted first so the summary/spill/preview can never carry base64 (I3). */ -function toText(result: unknown): string { +/** + * Render any result to the text the model would see (a string is itself; else compact JSON) and to the text + * its summary is made from. Inline media bytes are redacted first so the summary/spill/preview can never + * carry base64 (I3). A non-string result's summary text is its JSON once the walk has masked every member + * under a credential-named key and every argument after a credential flag whole (`redactSecretShapedWalk`), + * which no text rule reads. The text rules then read the JSON as one text (`makeSummary`), a value one JSON + * string down included, so a PEM block or a quoted value that runs across members keeps the context it is + * read in; projecting each string alone would mask a key or a PEM marker that a later member's text needs. + */ +function toText(result: unknown): { readonly text: string; readonly summaryText: string } { if (typeof result === 'string') { - return isBase64DataUri(result) ? '[base64 data URI omitted]' : result; + const text = isBase64DataUri(result) ? '[base64 data URI omitted]' : result; + return { text, summaryText: text }; } if (result === undefined) { - return ''; + return { text: '', summaryText: '' }; } + let safe: unknown; try { // Always run the redaction walk for a non-string result (inside the try so a throwing getter/proxy // degrades to '[unserializable]' rather than escaping): it strips inline base64 sources, base64 data: // URIs, AND raw binary buffers at any nesting before JSON-serialisation, so no media bytes (base64 or // decimal) can ride the summary/spill/preview (I3). A non-media object is returned as a structural clone - // (same JSON). JSON.stringify returns undefined for a value with no JSON form (a bare function/symbol); - // the `??` keeps a string without an `[object Object]`. - const safe = redactInlineMediaForText(result, new WeakSet()); - return JSON.stringify(safe) ?? '[unserializable]'; + // (same JSON). Both texts are made from this one walk. + safe = redactInlineMediaForText(result, new WeakSet()); + } catch { + return { text: '[unserializable]', summaryText: '[unserializable]' }; // throwing getter during redaction + } + return { + text: jsonText(() => safe), + summaryText: jsonText(() => + redactSecretShapedWalk( + safe, + new WeakSet(), + (text) => text, + undefined, + maskWholeKeepingPemMarkers, + ), + ), + }; +} + +/** + * JSON of a value, or `[unserializable]` for one that has no JSON form (JSON.stringify returns undefined for + * a bare function/symbol; the `??` keeps a string without an `[object Object]`) or whose serialization throws + * (a circular reference, a throwing toJSON). + */ +function jsonText(value: () => unknown): string { + try { + return JSON.stringify(value()) ?? '[unserializable]'; } catch { - return '[unserializable]'; // circular reference / throwing toJSON / throwing getter during redaction + return '[unserializable]'; } } function makeSummary(text: string): string { // Recognize LF/TAB-split shapes in the complete materialized text before one-line normalization. Clip only after recognition: a closing quote or URL // authority separator can be beyond the preview. Raw model/spill text stays unchanged. - const recognized = /[\n\t]/.test(text) ? redactSecretShapedText(text) : text; - const oneLine = redactSecretShapedText(recognized.replace(/\s+/g, ' ').trim()); + const recognized = /[\n\t]/.test(text) ? redactSecretShapedSummaryText(text) : text; + const oneLine = redactSecretShapedSummaryText(recognized.replace(/\s+/g, ' ').trim()); return oneLine.length <= SUMMARY_MAX ? oneLine : `${oneLine.slice(0, SUMMARY_MAX)}…`; } @@ -1151,8 +1546,8 @@ export async function boundForModel( host: ToolHost, signal?: AbortSignalLike, ): Promise { - const text = toText(result); - const summary = makeSummary(text); + const { text, summaryText } = toText(result); + const summary = makeSummary(summaryText); const bytes = utf8ByteLength(text); const lines = countLines(text); diff --git a/packages/core/src/tools/registry.test.ts b/packages/core/src/tools/registry.test.ts index 79451f1d..4fa3f304 100644 --- a/packages/core/src/tools/registry.test.ts +++ b/packages/core/src/tools/registry.test.ts @@ -29,8 +29,8 @@ import type { } from './types.js'; import { createInMemoryEffectJournal } from '../engine/execution-host.js'; -// Observe the journal projection, retaining its actual implementation, so one case can force a value -// that does not settle: real inputs that never settle are no longer known. +// Observe the journal projection, retaining its actual implementation, so a case can force a value that +// does not settle on arguments of its choosing; another uses a real input that does not settle. vi.mock('./bounding.js', async (importOriginal) => { const actual = await importOriginal(); return { ...actual, projectSecretShapedValue: vi.fn(actual.projectSecretShapedValue) }; @@ -1743,6 +1743,95 @@ describe('ToolRegistry — journal identity of credential-shaped arguments (PR95 } }); + it('names every argument when only the whole projection does not settle', async () => { + const actual = await vi.importActual('./bounding.js'); + vi.mocked(projectSecretShapedValue).mockImplementation((value) => ({ + value: actual.projectSecretShapedValue(value).value, + settled: !(typeof value === 'object' && value !== null && Object.keys(value).length > 1), + })); + try { + const writeFile = vi.fn['writeFile']>((path, data) => + Promise.resolve({ path, bytesWritten: data.length }), + ); + const host = stubHost(); + if (host.fs === undefined) throw new Error('missing stub fs'); + const context = ctx(); + const prepare = vi.fn(context.effects.prepare); + const err: unknown = await registry(stubHost({ fs: { ...host.fs, writeFile } })) + .dispatch(call('write_file', { path: './out.txt', content: 'body', append: true }), { + ...context, + effects: { ...context.effects, prepare }, + }) + .catch((error: unknown) => error); + expect(err, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').toBeInstanceOf(ToolArgsInvalidError); + if (!(err instanceof ToolArgsInvalidError)) throw new Error('missing refusal'); + expect(err.fields, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').toEqual([ + 'append', + 'content', + 'path', + ]); + expect(prepare, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').not.toHaveBeenCalled(); + expect(writeFile, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').not.toHaveBeenCalled(); + } finally { + vi.mocked(projectSecretShapedValue).mockImplementation(actual.projectSecretShapedValue); + } + }); + + it('refuses an open-schema effect whose argument names collide after masking', async () => { + // Two names that project alike would leave one member for two argument sets. + const dispatch = vi.fn(() => Promise.resolve({ receipt: 'published' })); + const tool: ToolDef = { + id: 'open_publish', + source: 'mcp', + description: 'Offline open-schema fixture', + llmVisibleParams: { type: 'object' }, + parseArgs: (raw) => z.record(z.string(), z.unknown()).parse(raw), + policy: { fsScoped: false, spawnsProcess: false, requiresGateApproval: false }, + effect: () => 3, + dispatch, + }; + const context = ctx({ grantedToolIds: new Set([tool.id]) }); + const prepare = vi.fn(context.effects.prepare); + const err: unknown = await createToolRegistry({ tools: [tool], host: stubHost() }) + .dispatch( + call('open_publish', { + 'sk-AAAAAAAAAAAAAAAAAAAA': 'rm -rf build', + 'sk-BBBBBBBBBBBBBBBBBBBB': 'echo done', + }), + { ...context, effects: { ...context.effects, prepare } }, + ) + .catch((error: unknown) => error); + expect(err, 'SECRETS_B8_CORE_KEY_COLLISION').toBeInstanceOf(ToolArgsInvalidError); + if (!(err instanceof ToolArgsInvalidError)) throw new Error('missing refusal'); + expect(err.fields, 'SECRETS_B8_CORE_KEY_COLLISION').toEqual(['[redacted]']); + expect(err.message.includes('sk-'), 'SECRETS_B8_CORE_KEY_COLLISION').toBe(false); + expect(prepare, 'SECRETS_B8_CORE_KEY_COLLISION').not.toHaveBeenCalled(); + expect(dispatch, 'SECRETS_B8_CORE_KEY_COLLISION').not.toHaveBeenCalled(); + }); + + it('refuses a real tiered effect whose arguments never settle', async () => { + // Four glued ids need four passes, one more than the projection's bound. + const ids = 'AKIAAAAAAAAAAAAAAAAAASIABBBBBBBBBBBBBBBB'.repeat(2); + const writeFile = vi.fn['writeFile']>((path, data) => + Promise.resolve({ path, bytesWritten: data.length }), + ); + const host = stubHost(); + if (host.fs === undefined) throw new Error('missing stub fs'); + const context = ctx(); + const prepare = vi.fn(context.effects.prepare); + const err: unknown = await registry(stubHost({ fs: { ...host.fs, writeFile } })) + .dispatch(call('write_file', { path: './keys.txt', content: `${ids}\n`, append: true }), { + ...context, + effects: { ...context.effects, prepare }, + }) + .catch((error: unknown) => error); + expect(err, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').toBeInstanceOf(ToolArgsInvalidError); + if (!(err instanceof ToolArgsInvalidError)) throw new Error('missing refusal'); + expect(err.fields, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').toEqual(['content']); + expect(prepare, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').not.toHaveBeenCalled(); + expect(writeFile, 'SECRETS_B7_CORE_UNSETTLED_REFUSAL').not.toHaveBeenCalled(); + }); + it('projects an exactly admitted-size quoted write after the write lands', async () => { const size = 8 * 1024 * 1024; const writeFile = vi.fn['writeFile']>((path, data) => diff --git a/packages/core/src/tools/registry.ts b/packages/core/src/tools/registry.ts index 06d8b6f8..13bdb635 100644 --- a/packages/core/src/tools/registry.ts +++ b/packages/core/src/tools/registry.ts @@ -342,16 +342,17 @@ async function prepareEffect( const journal = projectInput(def, effective, ctx.secretArgKeys, 'journal'); if (journal.unsettled.length > 0) { // A value whose redacted projection never settles projects to `[redacted]`, which any other such - // value shares: its digest could replay one call's result for a different call (§4), so the - // effect is refused before anything is journaled. The model can correct the argument. An open - // schema lets the model choose an argument's name, so the names are masked like any other text. + // value shares, and two member names that project alike leave one member: either way the digest + // could replay one call's result for a different call (§4), so the effect is refused before + // anything is journaled. The model can correct the argument. An open schema lets the model choose + // an argument's name, so the names are masked like any other text. const fields = [...new Set(journal.unsettled.map(redactSecretShapedText))]; throw new ToolArgsInvalidError( def.id, fields, `tool \`${def.id}\`: credential-shaped text in ${fields.join( ', ', - )} has no stable redacted form, so this effect cannot be given a journal identity`, + )} has no stable, distinct redacted form, so this effect cannot be given a journal identity`, ); } try { diff --git a/packages/llm/src/scrub-secrets.test.ts b/packages/llm/src/scrub-secrets.test.ts index 19962875..3b8fd17e 100644 --- a/packages/llm/src/scrub-secrets.test.ts +++ b/packages/llm/src/scrub-secrets.test.ts @@ -1297,6 +1297,7 @@ describe('PR95 review round 5: label pieces, a split ftp credential and the repe ['Uploading files\nftp://deploySECRET\n1234@host.example/x', '[REDACTED]'], ['Uploading files\n\tftp://deploySECRET\n1234@host.example/x', '[REDACTED]'], ['files\nf\ntp://TOKENSECRET@host', '[REDACTED]'], + ['Uploading files\tftp://deploySECRET\t1234@host.example/x', '[REDACTED]'], ]); // A real sftp scheme keeps its colonless user, also when the user is split. for (const input of [ @@ -1362,3 +1363,82 @@ describe('PR95 review round 6: the label bound behind a terminal sequence, fragm expect(scrubSecrets('public text'), 'SECRETS_B6_FAIL_CLOSED').toBe('[REDACTED]'); }); }); + +describe('PR95 review round 8: the split probe behind a terminal introducer and the host after a query', () => { + it('blanks a split credential whose first letter a bare terminal introducer would take', () => { + // Normalization reads ESC '[' plus the credential's first letter as a terminal sequence. + for (const input of [ + '\u001b[sk-Abc4Def5\nGhi6Jkl7Mno8Pqr9', + 'auth failed: \u001b[api-k\ney: Abc4Def5Ghi6Jkl7Mno8', + '\u001b[https://user:Abc4Def5\tGhi6Jkl7Mno8@host.example/x', + // A two-byte escape takes the letter too. + '\u001bAIzaSyAbc12345\n678901234567890xyz', + ]) { + expect(scrubSecrets(input), 'SECRETS_B8_SPLIT_RAW').toBe('[REDACTED]'); + } + const benign = 'plain \u001b[1mbold\u001b[0m text\nnext line'; + expect(scrubSecrets(benign), 'SECRETS_B8_SPLIT_RAW').toBe(benign); + }); + + it('reads the host after an ambiguous authority up to its query or fragment', () => { + for (const [input, expected] of [ + ['https://gw?a=1@host.example?q=1', 'https://[REDACTED]@host.example?q=1'], + ['https://gw?a=1@host.example#frag rest', 'https://[REDACTED]@host.example#frag rest'], + ] as const) { + expect(scrubSecrets(input), 'SECRETS_B8_HOST_QUERY').toBe(expected); + } + }); + + it('masks a header token that a JSON escape begins', () => { + // The escape's letter starts the token, so the value glued to it is masked with it. + for (const [input, expected] of [ + [ + '400 invalid header value: {"x-api-key":"\\nsk-ant-api03-Q9zXw7TokenValue12345abcdef"}', + '400 invalid header value: {"x-api-key":"\\[REDACTED]"}', + ], + [ + '{"x-goog-api-key":"\\tAIzaSyQ9zXw7TokenValue12345abcdef01"}', + '{"x-goog-api-key":"\\[REDACTED]"}', + ], + ['{"api-key":"\\r\\nQ9zXw7TokenValue12345abcdef"}', '{"api-key":"\\[REDACTED]"}'], + ['x-goog-api-key:\\n\\n\\nQ9zXw7TokenValue12345abcdef', 'x-goog-api-key:\\[REDACTED]'], + ['x-api-key:\\r\\n\\r\\nQ9zXw7TokenValue12345abcdef', 'x-api-key:\\[REDACTED]'], + [ + '{\\"api-key\\": \\"\\nQ9zXw7TokenValue12345abcdef\\"}', + '{\\"api-key\\": \\"\\[REDACTED]\\"}', + ], + ['api-key:"\\ nQ9zXw7TokenValue12345abcdef', '[REDACTED]'], + ] as const) { + expect(scrubSecrets(input), 'SECRETS_B8_HEADER_ESCAPE_TOKEN').toBe(expected); + } + }); + + it('blanks a path-less hyperlink target that the raw split probe joins with a later @', () => { + // A documented fail-closed trade-off: the raw LF/TAB-joined view reads a terminal hyperlink's bare + // origin and the next cell's address as one userinfo; a target with a path stops it. + const row = (target: string) => + `ID\tOWNER\n\u001b]8;;${target}\u001b\\build-42\u001b]8;;\u001b\\\tbob@example.com\n`; + expect(scrubSecrets(row('https://ci.example.com')), 'SECRETS_B8_HYPERLINK_TRADEOFF').toBe( + '[REDACTED]', + ); + const withPath = row('https://ci.example.com/builds/42'); + expect(scrubSecrets(withPath), 'SECRETS_B8_HYPERLINK_TRADEOFF').toBe(withPath); + }); + + it('masks a credential before or inside a terminal sequence in place', () => { + for (const [input, expected] of [ + [ + 'request failed: Authorization: Bearer Q9zXw7TokenValue\u001b[0m (status 401)', + 'request failed: Authorization: Bearer [REDACTED] (status 401)', + ], + [ + 'x-api-key: \u001b[33mabcdef123456\u001b[0m rejected by upstream', + 'x-api-key: [REDACTED] rejected by upstream', + ], + ['see https://user:pa\u001b[0mss@host.example/x', 'see https://[REDACTED]@host.example/x'], + ['Bearer \u001b[1mtoken123abc\u001b[0m done', 'Bearer [REDACTED] done'], + ] as const) { + expect(scrubSecrets(input), 'SECRETS_B8_TERMINAL_IN_PLACE').toBe(expected); + } + }); +}); diff --git a/packages/llm/src/scrub-secrets.ts b/packages/llm/src/scrub-secrets.ts index 5947c0e3..716d14a1 100644 --- a/packages/llm/src/scrub-secrets.ts +++ b/packages/llm/src/scrub-secrets.ts @@ -125,8 +125,11 @@ const NON_QUERY_SECRET_RULES: readonly SecretRule[] = [ // escaped, glued behind punctuation, LF/TAB-split or a concatenated run of up to eight, as in // `BearerBearer`), as a doubled `Bearer Bearer` misconfiguration does; the token after them is // masked and the labels stay. The token may begin with any run of (escaped) quotes, is never a - // label, a run of labels or another header name, and ends at a delimiter, at escapes before one or - // before a header name glued to it, which then masks its own value. Every label repetition is + // label or a run of labels that whitespace or the end follows, nor another header name, and ends + // at a delimiter, at escapes before one or before a header name glued to it, which then masks its + // own value. A JSON escape after the `:` starts the token, so it takes a value glued to the + // escape (`"x-api-key":"\n"` keeps only the backslash) but not a quoted value behind one + // (`x-api-key:\t""`), a documented limit (error-handling.md). Every label repetition is // bounded so no label run can exhaust the RegExp stack: past eight separate labels the standalone // label rule masks the token after the last one, and nine concatenated labels blank the diagnostic // (`LABEL_RUN_PAST_BOUND`). @@ -141,12 +144,12 @@ const NON_QUERY_SECRET_RULES: readonly SecretRule[] = [ // finite scheme LABEL, a label may be closed by its own quote before the whitespace // (`"Bearer" `), and a run of up to eight concatenated labels (`BearerBearer`) counts as one // label. Preserve the required label/value separator and ordinary following prose; opaque value - // fragments cannot be distinguished from that prose. A label never takes another label as its - // token, even one glued behind punctuation, quoted or concatenated, so any number of stacked labels - // leaves the last one to mask the token, and a token ends before a label (or a concatenated run) - // glued to it behind a non-word character, which then masks its own token. Quotes and `,;}` - // around a token stay outside it unless it continues past them, so `[REDACTED];tail` is masked - // whole. + // fragments cannot be distinguished from that prose. A label never takes as its token another + // label that whitespace or the end follows, even one glued behind punctuation, quoted or + // concatenated, so any number of stacked labels leaves the last one to mask the token, and a + // token ends before a label (or a concatenated run) glued to it behind a non-word character, + // which then masks its own token. Quotes and `,;}` around a token stay outside it unless it + // continues past them, so `[REDACTED];tail` is masked whole. { pattern: /\b(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)(?:[\\"']*(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)){0,7}[\\"']*\s+(?![^\w\s]*(?:(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)[\\"']*){1,8}(?:\s|$))[\\"']*(?(?=[^\s\\"'])\S*?[^\s\\"',;}])[\\"',;}]*(?=(?<=[^\w])(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)(?:[\\"']*(?:B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t]*a[\n\t]*s[\n\t]*i[\n\t]*c)){0,7}[\\"']*(?:\s|$)|\s|$)/dgi, @@ -409,6 +412,11 @@ function scrubOnce(text: string): string { const normalized = normalizeSecretProbe(text); if (LABEL_RUN_PAST_BOUND.test(normalized) || LABEL_RUN_PAST_BOUND.test(text)) return MASKED; const joined = /[\n\t]/.test(normalized) ? normalized.replace(/[\n\t]+/g, '') : undefined; + // Normalization can take a split credential's first letter into a terminal sequence (ESC 'A' + // before `AIza`, ESC '[' before `sk-`), so the raw text's joined view is probed as well when the + // two views differ. + const rawJoined = + normalized !== text && /[\n\t]/.test(text) ? text.replace(/[\n\t]+/g, '') : undefined; for (const rule of NON_QUERY_SECRET_RULES) { const rawMasked = applyRule(text, rule); // Identical inputs have identical pure projections; unchanged raw text already has @@ -417,6 +425,7 @@ function scrubOnce(text: string): string { if ((rawMasked === text ? normalized : normalizeSecretProbe(rawMasked)) !== normalizedMasked) return '[REDACTED]'; if (joined !== undefined && hasSplitCredential(normalized, joined, rule)) return '[REDACTED]'; + if (rawJoined !== undefined && hasSplitCredential(text, rawJoined, rule)) return '[REDACTED]'; // Retain the existing broader whitespace probe only within this rule's still-unmasked spans. // It detects incomplete prefixes separated by spaces without consuming prose after a full mask. for (const remaining of normalizedMasked.split('[REDACTED]')) { diff --git a/tools/causal-controls/provenance.test.mjs b/tools/causal-controls/provenance.test.mjs index 951565c7..8a082124 100644 --- a/tools/causal-controls/provenance.test.mjs +++ b/tools/causal-controls/provenance.test.mjs @@ -282,10 +282,10 @@ test('every declared filesystem and secrets anchor matches current production so }); test('secrets workload corrections retain independent complete-count and reader-witness pins', () => { - assert.equal(secretsControls.length, 120); + assert.equal(secretsControls.length, 130); assert.equal( secretsControls.reduce((count, control) => count + 1 + (control.witnesses ?? []).length, 0), - 501, + 513, ); for (const [id, file, expectedTests] of [ ['run-default-secret-reader', 'apps/cli/src/commands/run.test.ts', 84], @@ -338,16 +338,28 @@ test('secrets workload corrections retain independent complete-count and reader- 'core-key-value-through-masks', 'core-scheme-nested-label', 'core-mask-boundaries', - // Round 7: userinfo ownership, the quoted-value scan, chaining, the summary and the journal. + // Round 7: userinfo ownership, the quoted-value scan, the summary and the journal. 'core-userinfo-username-zone', 'core-split-zone-cap', 'core-quoted-value-scan', - 'core-standalone-chain', - 'core-glued-span-boundary', 'core-summary-fixed-point', 'core-journal-settled', - ].map((id) => [id, ['packages/core/src/tools/bounding.test.ts'], 195]), - ['core-unsettled-refusal', ['packages/core/src/tools/registry.test.ts'], 112], + // Round 8: structured summaries, escaped and unclosed quotes, continuation labels. + 'core-summary-walk', + 'core-escaped-quote', + 'core-unclosed-run', + 'core-unclosed-closing-view', + 'core-continuation-label', + // Pre-review: the summary's closer and whole-JSON reading, colliding names, values one JSON level down. + 'core-summary-through-closer', + 'core-summary-whole-json', + 'core-journal-key-collision', + 'core-escaped-value-start', + // Sixth pre-review: a summary's escaped separators and the PEM markers its walk keeps. + 'core-summary-escape-view', + 'core-summary-pem-markers', + ].map((id) => [id, ['packages/core/src/tools/bounding.test.ts'], 210]), + ['core-unsettled-refusal', ['packages/core/src/tools/registry.test.ts'], 115], ...[ 'diagnostic-joined-before-masking', 'diagnostic-joined-original-boundary', @@ -394,9 +406,10 @@ test('secrets workload corrections retain independent complete-count and reader- 'diagnostic-userinfo-authority-rest', 'diagnostic-userinfo-rest-length', 'diagnostic-url-split-retry', + 'diagnostic-split-probe-raw', 'diagnostic-label-run-normalized', 'diagnostic-label-run-raw', - ].map((id) => [id, ['packages/llm/src/scrub-secrets.test.ts'], 219]), + ].map((id) => [id, ['packages/llm/src/scrub-secrets.test.ts'], 224]), ...['diagnostic-inline-projection-order', 'diagnostic-original-control-boundary'].map((id) => [ id, ['apps/cli/src/render/sanitize.test.ts', 'apps/cli/src/process/render-error.test.ts'], diff --git a/tools/secrets-controls/README.md b/tools/secrets-controls/README.md index ace38bc4..1b312873 100644 --- a/tools/secrets-controls/README.md +++ b/tools/secrets-controls/README.md @@ -4,7 +4,7 @@ Step 4 first-run input matrix, Step 5 snapshot/export, Step 7 provider-header and PR #95 B2 redaction controls on physical source copies. `--mechanism ` supports diagnosis only; selected runs retain `complete: false`. The current manifest -declares **120 independent controls and 501 required named assertion witnesses**. +declares **130 independent controls and 513 required named assertion witnesses**. These counts describe the workload; a declaration is not evidence that its removal passed. Actual fixed/removal/restored results belong to the coordinator's summary. @@ -178,9 +178,8 @@ modern `sk-` body characters, joined-span application, the original-text left-bo check, split scheme labels, split API-header labels and raw `?`/`#` password recognition independently. Four LLM controls remove the before-masking LF/TAB probe, its original left-boundary check, split API-header recognition and raw-password recognition. -Every core projection phase executes the complete 195-case bounding file (the journal-identity -control's phases the complete 112-case registry file); every new LLM phase executes the complete -219-case scrubber file. Independent ordinary tooling tests pin +Every core projection phase executes the complete 210-case bounding file (the journal-identity +control's phases the complete 115-case registry file); every new LLM phase executes the complete 224-case scrubber file. Independent ordinary tooling tests pin these inventories, the 278-case overlap cohort, 180-case CLI diagnostic cohort and 18-case command-export cohort. Actual collection still has to match in every phase. @@ -191,8 +190,9 @@ would survive its removal. Other witnesses require complete recognized spans and retain public tails, host contacts and the full-value identity policy. The finite LF/TAB continuation and malformed-userinfo policy is documented in the canonical [effect-journal contract](../../docs/reference/shared-core/effect-journal.md#11-secrets-what-a-row-may-hold). -An eight-MiB permanent fixture checks completion and whole-span behavior without a -machine-speed threshold; these controls do not certify arbitrary obfuscation. +Eight-MiB permanent fixtures in the core and scrubber files check completion and whole-span +behavior under a 15-second case deadline rather than a machine-speed threshold; these controls do +not certify arbitrary obfuscation. Two R1 original-boundary recovery controls add eight named assertions, retaining all 51 prior identities and required witnesses. The core removal admits embedded-word @@ -261,20 +261,20 @@ previous-word prefilter and the rule's own boundary predicate; retaining either would leave the tested policy active. Header/key recovery removals still change their original-rule boundary only, and all removals retain the same canonical matchers and bounded raw/joined standalone visitors. Current collection is 26 -complete files / 1,617 cases, with each control's exact workload and named target +complete files / 1,640 cases, with each control's exact workload and named target checked separately. These are updated declarations and author checks, not new physical causal captures or a clean review round. R3 is preserved as NONCLEAN; later corrective acceptance belongs to the owning phase register. Prior selected `7e9096a8` captures retain their older source/workload attribution. -The current 120-control / 360-phase full declaration runs as two round-robin shards of 60 -controls (`--shard 1/2` and `--shard 2/2`), each exporting its own evidence: each projects 1,087 -retained files (60 × 18 + 7) of the unchanged 2,000. Locally both exports held exactly that many -and measured 45.50 MiB and 45.48 MiB, final export record included, against the unchanged 64-MiB -bound. That is one local measurement at the round-7 source, not a CI capture; one export of the -whole family would exceed the byte bound. Historical 41-control captures retain their -original source and cohort attribution; selected B2 or original-boundary recovery captures are not full-family -acceptance. +The current 130-control / 390-phase full declaration runs as two round-robin shards of 65 controls +(`--shard 1/2` and `--shard 2/2`), each exporting its own evidence: each projects 1,177 retained +files (65 × 18 + 7) of the unchanged 2,000. Locally both exports held exactly that many and measured +49.83 MiB and 49.76 MiB, final export record included, against the unchanged +64-MiB bound. That is one local measurement at the round-8 source, not a CI capture; one export of +the whole family would exceed the byte bound. Historical 41-control captures retain their original +source and cohort attribution; selected B2 or original-boundary recovery captures are not +full-family acceptance. Wave 3's PR95 increment adds nine controls and 80 named assertions, retaining all 57 prior identities and witnesses. The LLM header control restores the old header value @@ -417,6 +417,96 @@ and line-break rules, the refused argument names' masking and the helper's settl label join's `:` prefilter only skips work whose result is the text unchanged, so the output cannot show it and it has neither a test nor a control. +The eighth round found that round 7's glued-credential chain could start a narrower family inside a +wider credential or swallow a key name or label after it, and that four older gaps remained. The +chain and its glued-span exemption are gone: a glued credential is read at the earlier mask's +boundary on the next pass, and a run that never settles is redacted and refused. A structured +result's summary is read from its JSON as one text once credential members and flag arguments are +masked whole; quoted values are read one JSON string down, also behind escaped tabs and line breaks, +where an apostrophe opens a value too; a quoted value that does not close on its line, or one JSON +string down before its string ends, is decided once the pass's other masks are known, masked whole +when they join its lines and it closes and otherwise read as an unquoted run from after its quote, +no further than that end, while a summary masks one stopped by a line break through a later quote +that could close it (else to the end of its line) and one that reaches the end of its string or of +the text to that end; two member names that project alike make the journal projection unsettled; a +header label at a line start inside a split continuation is read through a second label view; and +the scrubber's split probe also reads the raw text when terminal normalization changed it. The +round's password-position finding is documented rather than changed: such a key's value runs to the +key rule's delimiter, through a URL's host and path, and calls that differ only there share a +projection, as recognized credentials do. Before the commit the author ran the reviewers' method on +the candidate, and six internal pre-reviews reviewed it. The first found that the draft's +alternatives each opened a gap, and they were dropped: an authority-end rule that trusted a +host-shaped tail revealed a password tail BASE had masked, quoted values that crossed lines took a +quote that closes a string for one that opens a value, and a line-start credential inside a +continuation made the split scan quadratic. The second found that the summary's pre-pass could erase +the key of a nested multi-line value and that JSON escaping hid such values in structured results, +that a value one JSON string down ran across a raw line break behind a backslash, and that two +member names projecting alike dropped a member from the journal identity. The third found that the +round's own additions failed: masking a value that never closes to the end of its line, decided from +the end of the text back, merged the journal identities of calls whose text after a quote that +closes a string around the key differed, so such a value is read as an unquoted run instead (the +author's distinctness oracle then found the same merge one JSON string down, where a value ran to +the end of its string, which now ends its run too); and the scrubber's reading of JSON-escaped tabs +after a header's colon and its view of terminal sequences' final bytes each let a token leak or +blanked coloured diagnostics, so both are withdrawn and their cases documented as limits. The fourth +found that the run, deciding a value in the pass whose other rule also masked its line break, and +the opaque mask's kept backslash showed values that every earlier version masked whole in its next +pass, so such a value is now decided once the pass's other masks are known and the opaque mask takes +the backslash again (the value one JSON string down that needed it is a documented limit); it also +found that a terminal hyperlink to a bare origin before a later `@` blanks a diagnostic through the +raw split probe, now a documented trade-off. The fifth found that projecting a structured result's +strings one by one masked the keys and PEM markers that reading its JSON needs across members, so a +structured summary reads its JSON as one text again and the per-string cache and summary-only runs +over line breaks are withdrawn. The sixth found that the closing view and the summary's closer read +a quote's escape after a nested key's value had taken the backslash before it, which showed a +value's tail that base had masked; that a structured summary's walk took a private-key marker that a +block across members needs; and that a structured summary read no JSON-escaped line break or tab, a +gap base shares. A quoted value now closes at the later of two readings, with the masks before and +inside it filled and with each mask keeping the backslashes it ends with, which also masks the tail +that an opaque value's mask unescaped, behind a URL userinfo (which base masked) or not (which base +showed too); the walk keeps a marker whose partner lies outside its member; and every summary also +reads a view in which each escaped line feed, carriage return or tab is that separator. A structured +summary's walk also became linear in its members at a small constant factor (one media-redaction +walk serves both texts, that walk applies no text rule to a string because the text rules read the +masked JSON once, and the key and label rules skip text that cannot hold them), and a many-part +result takes a case deadline. Core's four eight-MiB cases take the scrubber's case deadline in place +of their wall-clock bounds. `core-standalone-chain` and `core-glued-span-boundary` are removed with +their mechanisms; `core-quoted-value-scan`, `core-joined-original-boundary`, +`core-summary-fixed-point`, `core-key-value-through-masks`, `core-key-value-nested` and +`core-journal-settled` are re-anchored; twelve controls are added (`core-summary-walk`, +`core-escaped-quote`, `core-unclosed-run`, `core-unclosed-closing-view`, `core-continuation-label`, +`core-summary-through-closer`, `core-summary-whole-json`, `core-journal-key-collision`, +`core-escaped-value-start`, `core-summary-escape-view`, `core-summary-pem-markers`, +`diagnostic-split-probe-raw`); and `core-unsettled-refusal` gains an unmocked witness and one for +its whole-projection fallback. Pinned by tests without a control, each killed by its mutation: the +separator before an escaped quote in the key rule; the escaped scan's decoding, its raw line breaks, +also behind a backslash, and the raw quote that ends its string; a plain value's CR and backslash +line ends, a lone backslash at the end of the text and the end of the text that leaves a value open; +the run's start after its quote, its limit at its line's end, its six-character floor and its +delimiters, which unquoted values share, the closing view's reread, its guard on a quote that +another range takes and its second reading, the unquoted values' shared delimiter scan, the key +rule's second reading of a quoted value, the backslashes each mask keeps in it and the later of the +two readings; the raw tab, escaped CR, escaped apostrophe and raw apostrophe behind escaped +separators; the summary's closers (the masked values' coverage, the nearest outside closer, the +furthest masked one, the fill, the text end, the order by end, an escaped value's quote kind, the +closers' backslash parity, even, 4k+1 and 4k+3, and the second reading's closers, masks and escapes +with the later of the two), its fail-closed value, the summary projection in both passes, and the +undefined and unserializable results; the conditions of the key and scheme rules' early exits and of +the second scheme-label read, whose removal is output-neutral; the collision report inside arrays +and under keys; the second label view's first-separator fill, the fill's opening bracket and the `'` +delimiter; the structured summary's walk-only values; the escaped view's line feed, carriage return +and tab, its doubled separator, its run of backslashes and each of the three rules that read it; the +kept markers' first END and last BEGIN, the conditions under which a marker's partner lies inside +its member, and the whole mask through arrays, members and both spellings of a flag argument, with +the journal's and the display's own; the scrubber's host cut at a query or a fragment and the TAB +half of its split retry; and the registry's masked, deduplicated refused names and its +whole-projection fallback. Two and three PEM blocks in one text pin the shared END search's +re-search, whose removal loops until the test worker runs out of memory, so no control can certify +it. Four mutations are output-neutral and have neither a test nor a control: the kept markers' +`-----` prefilter and the two shortcuts that reuse a view when no mask ends with a backslash, which +only skip work, and preferring a second reading that closes at all, since a reading that keeps the +masks' backslashes closes only at a quote where the other reading has closed by then. + The thin entry point uses the same coordinator, outcome and bounded process join as the filesystem family in [causal-controls](../causal-controls/README.md). The provider-header diagnostic control pins the store's fixed throwable message diff --git a/tools/secrets-controls/mutations.mjs b/tools/secrets-controls/mutations.mjs index 5855b51f..424c48f3 100644 --- a/tools/secrets-controls/mutations.mjs +++ b/tools/secrets-controls/mutations.mjs @@ -254,16 +254,6 @@ const coreOriginalBoundary = String.raw` const candidates = /(^|[^\w])([A-Za-z] if (match === null) continue; matchedUntil = JOINED_STANDALONE_CREDENTIAL.lastIndex; mask(start, matchedUntil, '[redacted]'); - // A credential glued to the end of this one starts one of its own, as after an applied mask, so a - // run of fixed-length ids (AKIA…ASIA…) is masked whole in one pass. - for (let glued = matchedUntil; /[A-Za-z]/.test(joined[glued] ?? ''); glued = matchedUntil) { - JOINED_STANDALONE_CREDENTIAL.lastIndex = 0; - if (!JOINED_STANDALONE_CREDENTIAL.test(preview(glued))) break; - JOINED_STANDALONE_CREDENTIAL.lastIndex = glued; - if (JOINED_STANDALONE_CREDENTIAL.exec(joined) === null) break; - matchedUntil = JOINED_STANDALONE_CREDENTIAL.lastIndex; - mask(glued, matchedUntil, '[redacted]'); - } } } @@ -330,26 +320,21 @@ function visitSplitSpans( } return target + removed; } - let previousEnd = -1; visit(joined, (start, end, replacement, url) => { // A separator right before a URL's scheme is outside the URL, so it splits nothing. const schemeStart = url === undefined ? undefined : advance(url.schemeStart, true); const originalStart = advance(start, !ownsEdges); const ownsUntil = url === undefined ? undefined : advance(url.ownsUntil, ownsEdges); const originalEnd = advance(end, ownsEdges); - // A span that starts where the visitor's previous one ended is glued to a credential, which bounds - // it as an applied mask would. - const glued = originalStart === previousEnd; - previousEnd = originalEnd; - if (!glued && boundary?.test(text[originalStart - 1] ?? '')) return;`; + if (boundary?.test(text[originalStart - 1] ?? '')) return;`; const withoutCoreOriginalBoundary = coreOriginalBoundary .replace( String.raw` const candidates = /(^|[^\w])([A-Za-z])/g;`, String.raw` const candidates = /([A-Za-z])/g;`, ) - .replace(" if (!glued && boundary?.test(text[originalStart - 1] ?? '')) return;", ''); + .replace(" if (boundary?.test(text[originalStart - 1] ?? '')) return;", ''); /** The complete core projection test file every core projection control runs. */ -const CORE_PROJECTION_CASES = 195; +const CORE_PROJECTION_CASES = 210; function coreProjectionControl(id, before, after, targets, marker = coreProjectionMarker) { return control( id, @@ -364,9 +349,9 @@ function coreProjectionControl(id, before, after, targets, marker = coreProjecti ); } /** The complete registry test file the journal-identity control runs. */ -const REGISTRY_CASES = 112; +const REGISTRY_CASES = 115; /** The complete scrubber test file every LLM diagnostic control runs. */ -const LLM_SCRUB_CASES = 219; +const LLM_SCRUB_CASES = 224; function diagnosticControl(id, before, after, targets, marker = diagnosticMarker) { return control( id, @@ -1228,7 +1213,8 @@ export const mutations = [ ], 'SECRETS_B4_LABEL_RUN_BOUND', ), - // A header token is never a label, a run of labels or a nested header name. + // A header token is never a label or a run of labels that whitespace or the end follows, nor a + // nested header name. diagnosticControl( 'diagnostic-header-token-not-label', headerLine(), @@ -1496,15 +1482,15 @@ export const mutations = [ ), coreProjectionControl( 'core-key-value-nested', - " if (delimiterAt - valueStart >= 6) mask(match.index, delimiterAt, '[redacted]');", - " if (delimiterAt - valueStart >= 6) {\n mask(match.index, delimiterAt, '[redacted]');\n key.lastIndex = delimiterAt;\n }", + ' if (delimiterAt - valueStart >= 6) mask(match.index, delimiterAt);', + ' if (delimiterAt - valueStart >= 6) {\n mask(match.index, delimiterAt);\n key.lastIndex = delimiterAt;\n }', ["reads a key inside another key's value"], 'SECRETS_B6_CORE_NESTED_KEY', ), coreProjectionControl( 'core-key-value-through-masks', - ' visitSecretKeyValues(names, fillRanges(text, masked), record);', - ' visitSecretKeyValues(names, names, record);', + ' const values = fillRanges(text, masked);', + ' const values = names;', ['reads a value through the masks before it, line breaks included'], 'SECRETS_B6_CORE_VALUE_THROUGH_MASK', ), @@ -1549,8 +1535,8 @@ export const mutations = [ 'SECRETS_B4_CORE_SPLIT_EDGE', ), // Round 7: only a username owns its userinfo's text, and none past an LF or TAB; quoted values are - // scanned without a RegExp repeat; glued credentials chain; the summary uses the projection's fixed - // point; and an effect whose journal projection never settles is refused before it is prepared. + // scanned without a RegExp repeat; the summary uses the projection's fixed point; and an effect whose + // journal projection never settles is refused before it is prepared. coreProjectionControl( 'core-userinfo-username-zone', coreUserinfoZone, @@ -1567,38 +1553,26 @@ export const mutations = [ ), coreProjectionControl( 'core-quoted-value-scan', - ' const end = quotedValueEnd(values, valueStart);', - String.raw` const quoted = quote === '"' ? /"(?:[^"\\\r\n]|\\[^\r\n])*"/y : /'(?:[^'\\\r\n]|\\[^\r\n])*'/y; + ' const value = quotedValueEnd(values, valueStart);', + String.raw` const quoted = opening === '"' ? /"(?:[^"\\\r\n]|\\[^\r\n])*"/y : /'(?:[^'\\\r\n]|\\[^\r\n])*'/y; quoted.lastIndex = valueStart; - const end = quoted.test(values) ? quoted.lastIndex : -1;`, + const value = quoted.test(values) + ? { end: quoted.lastIndex, closed: true, lineBreak: false } + : { end: valueStart, closed: false, lineBreak: true };`, ['projects exactly admitted-size quoted values without exhausting the RegExp stack'], 'SECRETS_B7_CORE_QUOTED_SCAN', ), - coreProjectionControl( - 'core-standalone-chain', - " for (let glued = matchedUntil; /[A-Za-z]/.test(joined[glued] ?? ''); glued = matchedUntil) {", - ' for (let glued = matchedUntil; false; glued = matchedUntil) {', - ['masks a run of glued fixed-length ids in one pass, split or not'], - 'SECRETS_B7_CORE_CHAIN', - ), - coreProjectionControl( - 'core-glued-span-boundary', - ' const glued = originalStart === previousEnd;', - ' const glued = false;', - ['masks a run of glued fixed-length ids in one pass, split or not'], - 'SECRETS_B7_CORE_CHAIN', - ), coreProjectionControl( 'core-summary-fixed-point', - String.raw` const recognized = /[\n\t]/.test(text) ? redactSecretShapedText(text) : text;`, - String.raw` const recognized = /[\n\t]/.test(text) ? redactSecretShapedOnce(text) : text;`, + String.raw` const recognized = /[\n\t]/.test(text) ? redactSecretShapedSummaryText(text) : text;`, + String.raw` const recognized = /[\n\t]/.test(text) ? redactSecretShapedOnce(text, true) : text;`, ["summarizes with the projection's fixed point before one-line normalization"], 'SECRETS_B7_CORE_SUMMARY', ), coreProjectionControl( 'core-journal-settled', - ' settled = false;', - '', + " settled = false;\n return '[redacted]';", + " return '[redacted]';", ['reports whether a journal projection settled'], 'SECRETS_B7_CORE_JOURNAL_SETTLED', ), @@ -1611,6 +1585,96 @@ export const mutations = [ REGISTRY_CASES, 'refuses a tiered effect whose journal projection does not settle, before preparing it', 'SECRETS_B7_CORE_UNSETTLED_REFUSAL', + [ + 'refuses a real tiered effect whose arguments never settle', + 'names every argument when only the whole projection does not settle', + ].map((target) => ({ target, marker: 'SECRETS_B7_CORE_UNSETTLED_REFUSAL' })), + ), + // Round 8: a structured result is summarized from its projection, quoted values are read one JSON + // string down, one that does not close on its line is read as an unquoted run unless the pass's other + // masks join its lines, and a label may start at a line start inside a split continuation. + coreProjectionControl( + 'core-summary-walk', + ' const summary = makeSummary(summaryText);', + ' const summary = makeSummary(text);', + ['summarizes a structured result from its JSON once credential members are masked'], + 'SECRETS_B8_CORE_SUMMARY_WALK', + ), + coreProjectionControl( + 'core-escaped-quote', + ' const opening = escaped ? values[valueStart + 1] : values[valueStart];', + ' const opening = values[valueStart];', + ['reads a quoted value one JSON string down'], + 'SECRETS_B8_CORE_ESCAPED_QUOTE', + ), + coreProjectionControl( + 'core-unclosed-run', + " if (end - from >= 6) report(value.start, end, '[redacted]');", + '', + ['reads a quoted value that does not close on its line as an unquoted run'], + 'SECRETS_B8_CORE_UNCLOSED_QUOTE', + ), + coreProjectionControl( + 'core-unclosed-closing-view', + ' const closing = fillRanges(text, ranges);', + ' const closing = values;', + ['reads a quoted value that does not close on its line as an unquoted run'], + 'SECRETS_B8_CORE_UNCLOSED_QUOTE', + ), + // Pre-review: the summary masks a value that a later quote could close through that quote and reads + // a structured result's JSON as one text, a journal projection whose member names collide does not + // settle, one JSON level down a value opens behind escaped tabs and line breaks, and a value that + // does not close is decided once the pass's other masks are known. + coreProjectionControl( + 'core-summary-through-closer', + " return toFixedPoint((value) => redactSecretShapedOnce(value, true), text, '[redacted]');", + " return toFixedPoint((value) => redactSecretShapedOnce(value, false), text, '[redacted]');", + ['summarizes a value that closes on a later line whole'], + 'SECRETS_B8_CORE_SUMMARY_CLOSER', + ), + coreProjectionControl( + 'core-summary-whole-json', + ' (text) => text,', + ' redactSecretShapedSummaryText,', + ["reads a structured result's values across its line breaks and members"], + 'SECRETS_B8_CORE_SUMMARY_JSON', + ), + coreProjectionControl( + 'core-journal-key-collision', + ' if (Object.hasOwn(out, name)) collide?.();', + '', + ['reports a journal projection whose member names collide as unsettled'], + 'SECRETS_B8_CORE_KEY_COLLISION', + ), + coreProjectionControl( + 'core-escaped-value-start', + ' const valueStart = escapedValueStart(values, key.lastIndex);', + ' const valueStart = key.lastIndex;', + ['reads a quoted value one JSON string down'], + 'SECRETS_B8_CORE_ESCAPED_QUOTE', + ), + coreProjectionControl( + 'core-continuation-label', + ' for (const boundaries of new Set([afterMasks(text, masks), afterMasks(text, firstLines)])) {', + ' for (const boundaries of new Set([afterMasks(text, masks)])) {', + ['reads a label at a line start inside a split continuation'], + 'SECRETS_B8_CORE_CONTINUATION_LABEL', + ), + // Sixth pre-review: a summary reads the line breaks and tabs that JSON escaping hides, and its walk + // keeps the private-key markers a whole mask would take from a block that crosses members. + coreProjectionControl( + 'core-summary-escape-view', + ' if (throughCloser) visitEscapedSeparators(text, record);', + '', + ["reads a structured summary's JSON-escaped line breaks and tabs"], + 'SECRETS_B8_CORE_SUMMARY_ESCAPES', + ), + coreProjectionControl( + 'core-summary-pem-markers', + ' maskWholeKeepingPemMarkers,', + ' maskWhole,', + ['keeps the private-key markers a structured summary masks whole'], + 'SECRETS_B8_CORE_SUMMARY_PEM', ), coreProjectionControl( 'core-fixed-point', @@ -1619,6 +1683,13 @@ export const mutations = [ ['reaches a fixed point'], 'SECRETS_B4_CORE_FIXED_POINT', ), + diagnosticControl( + 'diagnostic-split-probe-raw', + " if (rawJoined !== undefined && hasSplitCredential(text, rawJoined, rule)) return '[REDACTED]';", + '', + ['blanks a split credential whose first letter a bare terminal introducer would take'], + 'SECRETS_B8_SPLIT_RAW', + ), diagnosticControl( 'diagnostic-url-split-retry', " /[\\n\\t]/.test(text[originalOffset + 1] ?? '') &&", From 59c64498a49efcbfbfdd718833e39094b21e27d2 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Wed, 21 Oct 2026 22:30:47 +0300 Subject: [PATCH 18/97] fix(core): address the ninth PR95 review round MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 9 of the PR95 increment's review (at 6f7dd342) confirmed 18 findings: two high, eight medium, seven low and a nit. Under the maintainer's 2026-10-21 acceptance bar every high and medium finding is corrected here; three lows are recorded in deferred-tasks.md. Core projection (packages/core/src/tools/bounding.ts): - Round 8 compared whole views with === inside its loops and walked a shared run of backslashes once per mask, both quadratic at admitted size (SEC-R9-1, SEC-R9-2). Each view of a quoted value is now built once per pass and its presence decided once, and keepingEscapes remembers each run's start at every end it serves. - A value whose line break a scheme token takes is reread with and without the key rule's own masks, so another key's value that takes its closing quote or ends at its opening one no longer keeps it open, and the key rule's first decision also reads views with the scheme labels' tokens filled, which only move a closing a plain reading reaches (R9-SCRUB-1, R9-GEN-1). - A summary masks a value that one reading stops at a line break through its later closer as well as through a reading that closes early (R9-SCRUB-2). - A structured summary reads the unwalked JSON once: each value the walk masks whole goes back where the walk left a private-use placeholder, as a whole JSON value or as the text of a --flag=value argument, and its mask is a range of that same pass. A quoted value or a private-key block that runs into or out of a credential member is masked across members again, and round 8's kept PEM markers are withdrawn (R9-SCRUB-3, R9-SCRUB-4, R9-GEN-2). - A summary reads private-key markers that a line break, a tab or more than one space splits; the journal reads them as before (R9-SCRUB-5, R9-GEN-3). Tests and controls: - Witnesses for every finding and an admitted-size case deadline for the four quadratic shapes in the journal, display and summary paths; an opaque parameter in a URL username keeps its host (TST-R9-1). A mutation run of 144 mutants kills 139; five are output-neutral. - core-summary-pem-markers, core-summary-whole-json and core-unclosed-closing-view are re-anchored, and core-key-value-through-masks now targets round 7's reading through earlier masks, since the unclosed reread also reads round 6's quoted witnesses; the family stays 130 controls / 513 targets and witnesses in two shards of 65, whose local exports held 1,177 files each and measured 50.19 MiB and 50.09 MiB. Docs: built-in-tools.md (the readings, the unwalked summary, wrapped markers, the 33-character glued-key window, and the terminal-sequence, indented-value and private-key block limits), effect-journal.md §11, deferred-tasks.md (TST-R9-2, R9-GEN-4, CRIT-R9-1, the freshness line, and two author-found private-key block gaps, unchanged since base, pending the maintainer's confirmation), the phase register's PR95 rows, round table, ruling text and gates, the secrets controls README and the status lines. The maintainer's round cap of 2026-10-21 makes round 10 the last: after it only high findings are corrected, and its other findings are recorded in deferred-tasks.md before Step 9a.1 begins. Refs: ADR-0113 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 6 +- CLAUDE.md | 6 +- docs/reference/shared-core/built-in-tools.md | 52 ++- docs/reference/shared-core/effect-journal.md | 2 +- docs/roadmap/current.md | 6 +- docs/roadmap/deferred-tasks.md | 41 +- .../phase-2.5.5-hardening-and-remediation.md | 23 +- packages/core/src/tools/bounding.test.ts | 235 ++++++++++-- packages/core/src/tools/bounding.ts | 350 ++++++++++++------ tools/causal-controls/provenance.test.mjs | 4 +- tools/secrets-controls/README.md | 41 +- tools/secrets-controls/mutations.mjs | 22 +- 12 files changed, 589 insertions(+), 199 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 26136003..c2f61143 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -126,9 +126,9 @@ gate (red since at least PR #90) was cleared on 2026-10-19. A pre-implementation and the maintainer's [2026-10-19 rulings](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) give every open PR #95 finding an owner and record 29 increments (no new ADR). The PR95 increment (the scrubber residuals, including both security residuals) is implemented on `development` and under -review: rounds 1–8 were NONCLEAN and are corrected ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)). Under the maintainer's -2026-10-21 masker acceptance bar, round 9 accepts it with no high or medium finding and no regression against -base; its low findings go to deferred-tasks.md. Then Step 9a.1 (release map/custody substrate, #46/ADR-0111). `development` is pushed after each accepted increment. No provider, +review: rounds 1–9 were NONCLEAN and are corrected ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)). Under the maintainer's +2026-10-21 masker acceptance bar and round cap, round 10 is the last: after it only high findings are +corrected, and its other findings go to deferred-tasks.md. Then Step 9a.1 (release map/custody substrate, #46/ADR-0111). `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. See [docs/roadmap/current.md](docs/roadmap/current.md) for live status. diff --git a/CLAUDE.md b/CLAUDE.md index 9d4faddc..2cb00ca6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -215,9 +215,9 @@ gate (red since at least PR #90) was cleared on 2026-10-19. A pre-implementation and the maintainer's [2026-10-19 rulings](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) give every open PR #95 finding an owner and record 29 increments (no new ADR). The PR95 increment (the scrubber residuals, including both security residuals) is implemented on `development` and under -review: rounds 1–8 were NONCLEAN and are corrected ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)). Under the maintainer's -2026-10-21 masker acceptance bar, round 9 accepts it with no high or medium finding and no regression against -base; its low findings go to deferred-tasks.md. Then Step 9a.1 (release map/custody substrate, #46/ADR-0111). `development` is pushed after each accepted increment. No provider, +review: rounds 1–9 were NONCLEAN and are corrected ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)). Under the maintainer's +2026-10-21 masker acceptance bar and round cap, round 10 is the last: after it only high findings are +corrected, and its other findings go to deferred-tasks.md. Then Step 9a.1 (release map/custody substrate, #46/ADR-0111). `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. For live status, per-PR history, milestone dates, and open obligations, see the canonical diff --git a/docs/reference/shared-core/built-in-tools.md b/docs/reference/shared-core/built-in-tools.md index c49a4261..16d4016c 100644 --- a/docs/reference/shared-core/built-in-tools.md +++ b/docs/reference/shared-core/built-in-tools.md @@ -148,9 +148,14 @@ where each mask keeps the backslashes it ends with, as the text has them. So a m backslash escaping a later quote neither closes the value early (`password="Bearer\nabcdefgh token=xyz\"tail" more` → `[redacted] more`) nor, when that quote is the value's only closer, leaves it open (`password="say \"…?sig=<24 or more characters>\"` → -`[redacted]`), in a summary too. One that does not close on its line is decided once the pass's other -masks are known: when one of them takes its line break (a scheme label's token, a nested key's value), -it closes as the next pass reads it and is masked whole, as before this rule; otherwise it is read as an unquoted value from after its quote — a run +`[redacted]`), in a summary too. Both readings are also taken with the scheme labels' tokens filled, +as the key rule read them before this projection merged its rules, so a token that takes the value's +line break lets it close at its real quote. One that does not close on its line is decided once the +pass's other masks are known: when one of them takes its line break (a scheme label's token, a nested +key's value), it closes as the next pass reads it and is masked whole, as before this rule, and it is +read with and without the key rule's own masks, so another key's value that takes its closing quote, +or ends at its opening one, does not keep it open (`password="a b Bearer\nxyzxyzxy token="q" more` → +`[redacted] more`); otherwise it is read as an unquoted value from after its quote — a run of at least six characters up to whitespace, a quote, `,`, `;` or `&` — and no further than its line's end, since its quote may instead close a string the key sits in: the text after such a quote stays (`stop logging "token=" values`), so calls that differ only there keep distinct projections, @@ -179,7 +184,9 @@ PEM block is masked whole. A key word must start within the first 33 characters from the last word boundary (a `-` starts one, a `_` does not), so a longer name is no key (`MY_APPLICATION_PRODUCTION_DATABASE_PASSWORD=…` stays). A key's mask starts where its word starts, so a credential glued to the front of a key name (`password=…`) is masked with that key's -value and stays when the value is empty or too short to mask. A key inside another key's value, and +value and stays when the value is empty or too short to mask; a glued token of 33 or more characters +puts the key word past that window, so no key is read and the value stays visible unless another rule +recognizes it (`ghp_<36 characters>password=…`). A key inside another key's value, and a scheme label inside another label's token, start a credential of their own (`token=a>password: ` → `[redacted]`; `Bearer a/Bearer ` → `Bearer [redacted] [redacted]`), and an LF/TAB-split header label or a scheme label may start right @@ -200,23 +207,34 @@ the admitted text, and none uses a backtracking pattern whose stack an admitted quoted value is read by a character scan no further than its line, and a summary decides its values that stop at a line break by one backward scan. Ordinary spaces and punctuation separate public prose; the masker does not discard the whole diagnostic because a credential appears. A summary uses -the projection with one difference, since a summary is shown, never compared: a quoted value that -stops at a line break is masked through the nearest later quote that could close it outside the -values the rule masks or, when every such quote is inside one, through the last of them, and only -else to the end of its line, in whichever of the two readings reaches further; one that reaches the end of the string around it or of the text without -closing is masked to that end. A structured (non-string) result is summarized from its JSON once -every member under a credential-named key and every argument after a credential flag is masked whole -(a count such as `input_tokens` too, the text rules then masking the member's name with it); such -a mask keeps a private-key BEGIN or END marker whose partner lies outside the member, so a block that -runs on into or out of it is masked across members. The text rules read that JSON as one text, a -value one JSON string down included, so a JSON-escaped line break is no delimiter and a PEM block or -a quoted value that runs across members keeps its key and markers; a member masked whole is not read -for a quoted value that runs on from it. Every summary also reads its text a second time with each +the projection with one difference, since a summary is shown, never compared: a quoted value that one +of its readings stops at a line break is masked through the nearest later quote that could close it +outside the values the rule masks or, when every such quote is inside one, through the last of them, +and only else to the end of its line, in whichever reading reaches further, and through the closing +of a reading that closes too; one that reaches the end of the string around it or of the text without +closing is masked to that end. A structured (non-string) result is summarized from its JSON with +every member under a credential-named key and every argument after a credential flag masked whole +(a count such as `input_tokens` too, the text rules then masking the member's name with it). The text +rules read that JSON once before those masks, which the same pass applies, so a quoted value or a PEM +block that runs into or out of a masked member is masked across members, as reading the unwalked JSON +masks it; they read the JSON as one text, a value one JSON string down included, so a JSON-escaped +line break is no delimiter and a PEM block or a quoted value that runs across members keeps its key +and markers. A summary also reads a private-key marker whose words a line break, a tab or more than +one space separates (`-----END RSA PRIVATE\nKEY-----`), as a block wrapped at a fixed width can split +one; the journal projection reads such a marker as before, and a structured result's summary does not +read one whose line break or tab its JSON escapes (`{"a":"…-----END RSA PRIVATE\nKEY-----"}`), a +documented limit. A private-key block is masked only from a BEGIN marker to an END marker that starts +at most 20,000 characters after it, so a block cut off before its END marker (`head -n 5 key.pem`) +stays visible in every projection, a documented limit. In a summary, a key's unquoted value behind +an escaped line break and indentation (`password:\n ` in a structured result's JSON) is not read, +a documented limit. Every summary also reads its text a second time with each escaped line feed, carriage return or tab (`\n`, `\r`, `\t`, behind any run of backslashes) as that separator, for the rules a separator decides: a credential or a scheme label right after one, and a credential split by one, are masked as in raw text (`{"stdout":"keys:\nsk-…"}` → `{"stdout":"keys:\n[redacted]"}`), while a header label split by one is not read; the journal -projection reads no escape. Supported +projection reads no escape. Core does not read a credential or a scheme label glued to a terminal +sequence's final byte (`ESC[1msk-…`), which leaves no word boundary there, a documented limit; the +backstop's terminal handling is in [error handling](../../standards/error-handling.md). Supported split shapes are recognized **before** complete-text one-line normalization, by that projection's full fixed point, followed by ordinary recognition and length clipping. This preserves multiline quoted-value recognition without exposing a split token's suffix. Public argument bytes are not diff --git a/docs/reference/shared-core/effect-journal.md b/docs/reference/shared-core/effect-journal.md index 7ab75130..b70284ea 100644 --- a/docs/reference/shared-core/effect-journal.md +++ b/docs/reference/shared-core/effect-journal.md @@ -394,7 +394,7 @@ credential of its own, and a value through earlier masks; masks userinfo split b either edge or inside its `scheme://`; reads a quoted value one JSON string down too, and one that does not close on its line, or before the end of its string, as an unquoted run that stops there — unless another mask of the pass takes its line break and it closes, as the next pass read it before -(at the later of its readings with and without the masks' trailing backslashes) — so the text after a quote that may close a string around the key keeps calls distinct; reads a +(at the latest of its readings, with and without the key rule's own masks and the masks' trailing backslashes) — so the text after a quote that may close a string around the key keeps calls distinct; reads a split header label or a scheme label right after a credential or a PEM block, and a header label at a line start inside a split continuation; and repeats the projection until it is stable. These change a `v5:` projection only for the arguments they project differently, such as userinfo diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index 1c872fc3..380be2df 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -165,9 +165,9 @@ Wave 3 implementation continues on `development`: Steps 1–8 are merged. A pre- of Steps 9–19 and the maintainer's rulings are recorded in the [2026-10-19 rulings](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) (no new ADR; 29 recorded increments). **The PR95 increment is implemented on `development` and under review** — the PR #95 -scrubber residuals, including the two security residuals; rounds 1–8 were NONCLEAN and are corrected, as its -[implementation checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20) records. Under the maintainer's 2026-10-21 masker acceptance bar, round 9 accepts it with no high or medium -finding and no regression against base, its low findings recorded in +scrubber residuals, including the two security residuals; rounds 1–9 were NONCLEAN and are corrected, as its +[implementation checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20) records. Under the maintainer's 2026-10-21 masker acceptance bar and round cap, round 10 is the last: after it +only its high findings are corrected, and its other findings are recorded in [deferred-tasks.md](deferred-tasks.md#pr-95-masker-low-edge-cases--approved-deferrals-2026-10-21). **Step 9a.1** follows, the first increment of the release map/custody substrate (#46, ADR-0111), whose 9a.1–9d increments carry a dedicated archive/crypto security sitting. Steps 10–19 follow the [increment table](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) and end with diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index 281e471f..4c47b6ac 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -2,7 +2,7 @@ > Status: Living -> Last updated: 2026-10-20 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Steps 9–19 not started; the PR95 increment is under review on `development`). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings. Existing approved deferrals remain. +> Last updated: 2026-10-21 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Steps 9–19 not started; the PR95 increment is under review on `development`). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's low edge cases with the maintainer's 2026-10-21 acceptance bar. Existing approved deferrals remain. > The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) records completed prerequisites; full M/N lifecycle and other scheduled work remain with their owners. > > Earlier triage: the Phase-2.6 rewrite triaged every open item, the 2026-07-19 full-project @@ -454,7 +454,8 @@ Under the [masker acceptance bar](phases/phase-2.5.5-hardening-and-remediation.m that the maintainer set on 2026-10-21, a low finding in the open-ended masker work (`packages/llm/src/scrub-secrets.ts` and core's twin projection in `packages/core/src/tools/bounding.ts`) is recorded here instead of being corrected in the review loop. None of these entries is a regression against base -`9902e0a0`, and none was judged high or medium. The shapes that neither masker reads by design stay described in +`9902e0a0`, and none was judged high or medium; the two entries marked pending were found by the author after +round 9, outside any finding, and await the maintainer's confirmation after round 10. The shapes that neither masker reads by design stay described in their canonical homes, [built-in-tools.md](../reference/shared-core/built-in-tools.md) for core and [error-handling.md](../standards/error-handling.md) for the scrubber; this list holds the review findings and the re-checks they need. **Trigger for every entry:** a real tool result, journal entry or diagnostic that carries the @@ -477,6 +478,42 @@ the repository, so each entry names the input and the property to re-check by ha journal digests, so it needs the compatibility review of [effect-journal.md](../reference/shared-core/effect-journal.md#11-secrets-what-a-row-may-hold) §11. *(low · packages/core/src/tools/bounding.ts)* +- [ ] **Approved deferral (maintainer, 2026-10-21). A summary does not read a key's unquoted value behind an + escaped line break and indentation.** *Low, unchanged from base.* *(round 9 critic, CRIT-R9-1)* + `boundForModel({ stdout: 'database:\n password:\n Qx7vT2mN9pL4kR8s\n user: app' })` shows the value, while + the same text as a string result is masked. The key rule reads a value behind escaped separators only when a + quote opens it (`escapedValueStart`), and the summary's escaped-separator view runs no key rule. **Re-check:** a + fix lets the summary's key rule read an unquoted value behind escaped separators and spaces, or runs it over + `escapedSeparatorView`; the journal is unchanged. *(low · packages/core/src/tools/bounding.ts)* +- [ ] **Approved deferral (maintainer, 2026-10-21). Core does not read a credential glued to a terminal + sequence's final byte.** *Low, unchanged from base.* *(round 9, R9-GEN-4)* `ESC[1msk-` and + `ESC[0mBearer ` keep the credential: the SGR final byte `m` is a word character, so the standalone and + scheme rules see no boundary; built-in-tools.md names the limit. **Re-check:** read those two rules in a view + where CSI sequences are filled, as the backstop's normalized probe does. *(low · packages/core/src/tools/bounding.ts)* +- [ ] **Approved deferral (maintainer, 2026-10-21). The quoted value's later-closing rule has tests but no causal + control.** *Low, partial.* *(round 9, TST-R9-2)* The mutations that keep a single reading, drop the kept + backslashes or skip the summary's second closer are each killed by + `closes a quoted value at the later of its readings with and without the masks`, but no declaration in + `tools/secrets-controls/mutations.mjs` targets it. **Re-check:** add a control (for example `keepingEscapes` + returning `ranges`) when the secrets family is next extended, with its counts. + *(low · tools/secrets-controls/mutations.mjs)* +- [ ] **Deferral pending the maintainer's confirmation (author, 2026-10-21). A private-key block cut off before + its END marker is not masked.** *Unchanged from base; found by the author after round 9, outside any review + finding.* `redactSecretShapedText('\nMIIE…')` and the summary of the same text keep the whole + body, as does a block whose END marker starts more than 20,000 characters after its BEGIN marker + (`PEM_BODY_MAX`): `visitPemBlocks` masks only a BEGIN…END pair, and the LLM backstop reads no private-key + block. A real tool output can cut a key short (`head -n 5 key.pem`, a tool that clips its own output). + built-in-tools.md names the limit. **Re-check:** mask an unclosed BEGIN marker through its base64 body lines in + every projection, linear at admitted size; a journal change needs the compatibility review of + [effect-journal.md](../reference/shared-core/effect-journal.md#11-secrets-what-a-row-may-hold) §11. + *(packages/core/src/tools/bounding.ts)* +- [ ] **Deferral pending the maintainer's confirmation (author, 2026-10-21). A structured result's summary does + not read a private-key marker split by a line break or tab.** *Unchanged from base; found by the author after + round 9.* `boundForModel({ a: '\nMIIE…\n-----END RSA PRIVATE\nKEY-----' })` shows the block in + its summary, while the same text as a string result is `[redacted]`: JSON escapes the line break or tab, so the + wrapped-marker patterns, which read raw whitespace, see none; more than one space still collapses in the + summary's one-line pass. built-in-tools.md names the limit. **Re-check:** read the wrapped markers in + `escapedSeparatorView` as well (`visitEscapedSeparators`). *(packages/core/src/tools/bounding.ts)* ## Phase 2.6.5 deferrals — `CR-93` and `CR-95` (long half), recorded 2026-09-06; `CR-70`, `CR-94` (remainder) and the in-turn overflow continuation, recorded 2026-09-14; the errored-or-aborted turn residual, the pre-upgrade free-page bytes and the estimator's calibration, recorded 2026-09-18 diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 82640a11..241cc71c 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -3694,7 +3694,7 @@ No credential values or request captures belong in tracked evidence. | 8b | **Recorded split — creation-owned native processes:** tool POSIX group versus MCP PID, actual close/retirement, verified absolute Windows teardown admission and helper lifetime; connect OS/editor helpers and replace independent reapers without claiming full descendant containment. | | 8c | **Recorded split — workflow and render integration:** run/gate/budget/driver owners, real return/throw health priority, cancellation, terminal/job-control leases and force-quit routing. No timer certifies DB close or receipt settlement. | | 8d | **Recorded split — sessions, terminal and activation:** fresh/resumed/one-shot/Home/pending/replaced session lifetimes, retained publication failure and shared actual cleanup promises; connect caught slash/rebuild faults, restore terminal custody before diagnosis, then activate early coordinator/background/sink integration. Actual native/platform/packed controls and complete cumulative Step 8 acceptance follow. | -| PR95 | **Recorded plan amendment, 2026-10-19 (maintainer-approved; new scope from the [PR #95 independent review](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18), not a split of rows 1–19) — scrubber residuals, before 9a:** the `api-key:Bearer`/`Basic` label erasure that leaks the token; raw `?`/`#` URL userinfo masked unless the text before it is host-shaped **and** an `=` assignment separates it from the authority `@`, applied to the LLM leaf and core's twin predicate; an extended finite scheme list (postgres, postgresql, mysql, mariadb, redis, rediss, mongodb, mongodb+srv, amqp, amqps, ssh, sftp, ldap, ldaps, smtp, smtps) masking colon-bearing userinfo for the new schemes; module-load checks of the split-credential scanner's two invariants, a start prefilter, joined-view and glued-scheme witnesses, the load-sensitive 8 MiB timing case, and the causal-control pins. One increment with ordered commits, two NEW clean complete cumulative rounds and a separate security sitting. | +| PR95 | **Recorded plan amendment, 2026-10-19 (maintainer-approved; new scope from the [PR #95 independent review](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18), not a split of rows 1–19) — scrubber residuals, before 9a:** the `api-key:Bearer`/`Basic` label erasure that leaks the token; raw `?`/`#` URL userinfo masked unless the text before it is host-shaped **and** an `=` assignment separates it from the authority `@`, applied to the LLM leaf and core's twin predicate; an extended finite scheme list (postgres, postgresql, mysql, mariadb, redis, rediss, mongodb, mongodb+srv, amqp, amqps, ssh, sftp, ldap, ldaps, smtp, smtps) masking colon-bearing userinfo for the new schemes; module-load checks of the split-credential scanner's two invariants, a start prefilter, joined-view and glued-scheme witnesses, the load-sensitive 8 MiB timing case, and the causal-control pins. One increment with ordered commits, two NEW clean complete cumulative rounds and a separate security sitting. *Acceptance amended on 2026-10-21 by the maintainer's masker acceptance bar and round cap ([checkpoint](#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)).* | | 9 | **Release map/custody substrate:** #46/ADR-0111, independently reviewed after the finite graph. Upstream TS maps, sole prepack final build, pre-hash trailer removal/staging, exact tarball/CLI+engine symbolication, encrypt-once candidate with permissioned prebuild discovery/reservation → pack/reuse → smoke → archive → npm → finalization. Required artifact/platform aggregation, all rerun modes, lost-staging refusal and permission-enforcing offline API controls. Dedicated archive/crypto security sitting. Fixture proof never claims production recipient/custody configuration. | | 9a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage #46/ADR-0111 unchanged, no new contract) — release maps, final build and symbolication:** fresh uncached upstream original-TypeScript maps, the sole prepack final CLI build, final trailer removal before hashes, map and private-graph staging outside public artifacts and caches, exact tarball-to-final-JS binding, and real packed CLI and engine frame symbolication with wrong JS/map/source and missing-map negative controls. Ordinary builds remain map-free. *Split again on 2026-10-19 into 9a.1 and 9a.2 below, which replace it as increments ([Plan shape](#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19)).* | | 9a.1 | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — candidate build and finalizer:** an owned isolated candidate copy, fresh uncached upstream maps, candidate-mode tsup selection with absent `sourcesContent`, the prepack finalizer (upstream and final map validation, repository-relative sources, exact trailer removal before hashes), owner-only private staging outside public artifacts and caches, closure/seal adaptation, the ordinary map-free control, the PR #95 archive/worker fold-ins, and a fail-closed `release.yml` publish guard that refuses any non-release candidate until 9d and Wave 4b custody exist. The receipt stays `compiled_package_test`. | @@ -9393,7 +9393,7 @@ plan amendment (row PR95) and the recorded splits; ID coverage is unchanged apar | Area | Ruling | | --- | --- | -| Acceptance | Each increment keeps the existing rule: implement, author gates, commit, then NEW independent complete cumulative review rounds until two consecutive rounds are clean after the last material fix. Comment- or documentation-only nits do not reset the count. Security-touching increments add a separate scoped sitting: PR95, 9a.1, 9a.2, 9b, 9c, 9d (whole Step 9 custody), 10a, 10c, 11b, 13c, 14b, 15c, 16a, 16b, 16c (whole Step 16), 18 and 19b (whole wave). | +| Acceptance | Each increment keeps the existing rule: implement, author gates, commit, then NEW independent complete cumulative review rounds until two consecutive rounds are clean after the last material fix. Comment- or documentation-only nits do not reset the count. Security-touching increments add a separate scoped sitting: PR95, 9a.1, 9a.2, 9b, 9c, 9d (whole Step 9 custody), 10a, 10c, 11b, 13c, 14b, 15c, 16a, 16b, 16c (whole Step 16), 18 and 19b (whole wave). PR95 instead follows the maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)). | | Publication | `development` is pushed after each accepted increment, so its actual CI (including the four same-tarball platform consumers) is observed per increment. The Wave 3 PR follows Step 19. | | Plan shape | PR95 + 9a.1, 9a.2, 9b, 9c, 9d + 10a–c + 11a–b + 12a–b + 13a–c + 14a–b + 15a–c + 16a–c + 17a–b + 18 + 19a–b (29 increments). Each split is recorded before its step starts. | | Reassignments | G54 moves from row 17 to 11a; #311's implementation moves to 10b and its certification stays in 17a; G36/G46 are re-scoped in 10a to the branch that replaces the raw-stack branch and are certified in 17b; promotion of the Steps 6–10 commands.md sections moves from Step 19 to 10c. | @@ -9443,6 +9443,7 @@ stay local and Git-ignored. | 6 | `6b2e94dd` | NONCLEAN | 21 (9 high) | round 5's value mirror was quadratic and still erased context, and its split view dropped the value mask; core now merges every rule's ranges on one text, as the scrubber does, reads a key or scheme label inside another value and a value through earlier masks, reads a split header label or a scheme label right after a credential, and scans PEM blocks linearly; the scrubber's nine-label bound also reads the raw text and its literal query names follow `#`; both fail-closed values are tested | | 7 | `b89ac62e` | NONCLEAN | 17 (2 high) | only a userinfo's username owns its keys, never past an LF or TAB, so a key in its password or parameter position or on the next line masks its value through the last `@`; quoted values are found by a character scan that an admitted 8-MiB value cannot exhaust; a credential glued to a fixed-length one is masked in the same pass; summaries use the projection's fixed point; a tiered call whose journal projection does not settle is refused before its prepare; the secrets controls run as two CI shards so that each evidence export fits the unchanged 64-MiB bound, which the single export had exceeded; untested pieces gained discriminating tests | | 8 | `082962e2` | NONCLEAN (integration and security clean) | 16 (1 high) | round 7's glued-credential chain cut wider credentials and swallowed key names, and is gone; a structured result's summary reads its JSON as one text once credential members are masked; quoted values are read one JSON string down, and one that does not close on its line is read as an unquoted run from after its quote, no further than that line's end (one JSON string down, its string's end), unless another mask of the pass takes its line break and it closes; a summary masks one stopped by a line break through a later quote instead; a header label at a line start inside a split continuation is read; the scrubber's split probe also reads the raw text; a password-position key's extent is documented as credential equality. Before the commit the author ran the reviewers' method on the candidate and six internal pre-reviews: the first found three of the draft's alternatives unsound (a trusted host-shaped tail, quoted values across lines, a quadratic line-start scan), and they were dropped; the second (17 findings, none high) led to the summary's through-the-closer masking, colliding member names refused as unsettled and escaped separators and opaque values before escaped quotes; the third (27 findings, two high) found that the round's own additions failed — masking a value that never closes to its line end merged journal identities, and the scrubber's escaped header separator and terminal final-byte view let a token leak or blanked coloured diagnostics — so such a value is read as an unquoted run, the two scrubber additions are withdrawn and documented as limits, and a structured summary's walk became linear at a small constant factor; the fourth (16 findings, two medium; the critic clean, with all 128 controls run) found that the run and the opaque mask's kept backslash showed values that earlier versions masked whole in their next pass, so such a value is decided once the pass's other masks are known and the opaque mask takes the backslash again, and gaps in tests and wording; the fifth (19 findings, one high; the critic clean) found that projecting a structured result's strings one by one removed the PEM markers and keys that reading its JSON needs across members, so a structured summary reads its JSON as one text again and the summary-only runs over line breaks are withdrawn; the sixth (four findings and a duplicate, one medium; the integration lens and the critic clean) found that the closing view and the summary's closer read a quote's escape after a nested key's value had taken its backslash, and that a structured summary's walk dropped the private-key markers a block across members needs, both regressions against base, and that a structured summary read no JSON-escaped line break or tab, a gap base shares, so a quoted value closes at the later of its readings with and without the masks' trailing backslashes, which also masks the tail an opaque value's mask unescaped (behind a URL userinfo, a further regression the author's oracles found), the walk keeps a marker whose partner lies outside its member, and every summary also reads its escaped separators; two low edge cases base shares went to deferred-tasks.md under the maintainer's 2026-10-21 bar | +| 9 | `6f7dd342` | NONCLEAN (integration clean) | 18 (2 high, 8 medium) | round 8's view comparisons and per-range backslash walks were quadratic at admitted size, so each view is built once and a shared backslash run is read once; a quoted value is also read with the scheme labels' tokens filled, and one that does not close on its line with and without the key rule's own masks; a summary masks a value that one reading stops at a line break through its later closer; a structured summary reads the unwalked JSON once with the walk's whole masks applied in that pass, which replaces round 8's kept PEM markers; a summary reads private-key markers that a line break, a tab or more than one space splits; three low findings went to deferred-tasks.md, and two private-key block gaps the author found afterwards, unchanged since base, are named as limits and recorded there pending the maintainer's confirmation | **Author gates and controls.** Every corrected head passed `pnpm run ci` and the affected suites under `CI=true GITHUB_ACTIONS=true`, plus local differential, exhaustive and admitted-size harnesses @@ -9456,7 +9457,7 @@ project 1,087 of the unchanged 2,000 retained files; both shards ran locally wit `RELAVIUM_ACCEPTANCE_EVIDENCE` set, and each export held 1,087 files and measured 45.50 MiB and 45.48 MiB, its final record included. From round 8 the family is 130 controls / 390 phases / 513 targets and witnesses in two shards of 65 controls (1,177 files each), whose exports measured -49.83 MiB and 49.76 MiB. From round 8 the author also runs the reviewers' own +49.83 MiB and 49.76 MiB (50.19 MiB and 50.09 MiB at the round-9 source). From round 8 the author also runs the reviewers' own generative harness (a planted-token oracle, idempotence, JSON round trips, prose preservation, admitted-size timing and a search for projections that do not settle) and an internal multi-lens pre-review on every correction before its commit. @@ -9468,7 +9469,8 @@ discriminating tests rather than by new controls. internal pre-reviews, the maintainer narrowed this increment's acceptance bar. A pattern-based masker cannot be complete, and each aggressive round found another corner case while Step 9 waited. A round is clean for this increment when it reports no high or medium finding and no regression against base -`9902e0a0` (text that base masked and the reviewed head shows). Each low finding is recorded in +`9902e0a0` (text that base masked and the reviewed head shows, or a guarantee base kept, such as no +throw, linear time at admitted size or a settled projection, that the reviewed head no longer keeps). Each low finding is recorded in [deferred-tasks.md](../deferred-tasks.md#pr-95-masker-low-edge-cases--approved-deferrals-2026-10-21) with its file, a minimal example and where to re-check it, instead of being corrected in the loop. The increment closes with the sixth internal pre-review and one external round (round 9) under that @@ -9477,7 +9479,14 @@ twin projection in `packages/core/src/tools/bounding.ts` and their controls. Eve keeps the approved bar of two NEW consecutive clean complete cumulative rounds after the last material fix. -**Remaining gates.** The increment is not independently accepted. Round 9 remains: a NEW complete -cumulative round, with its separate security lens, that meets the 2026-10-21 bar. Then `development` -is pushed and its CI observed, and the PR #95 items in +**Maintainer ruling — the round cap, 2026-10-21.** Round 9 reported two high findings, in round 8's +own quadratic view comparisons, and eight medium regressions in five constructed shapes. The maintainer +ruled that every round-9 finding is corrected and round 10 runs, and that the loop then stops: after +round 10, whatever it reports, only its high findings are corrected; its medium and low findings are +recorded in deferred-tasks.md with the file, a minimal example and where to re-check it, and the plan +moves on to Step 9a.1. + +**Remaining gates.** The increment is not independently accepted. Round 10 remains: a NEW complete +cumulative round with its separate security lens, under the round cap. Then `development` is pushed +and its CI observed, and the PR #95 items in [deferred-tasks.md](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18) close. diff --git a/packages/core/src/tools/bounding.test.ts b/packages/core/src/tools/bounding.test.ts index 3406b9c0..56ca0073 100644 --- a/packages/core/src/tools/bounding.test.ts +++ b/packages/core/src/tools/bounding.test.ts @@ -2652,11 +2652,12 @@ describe('PR95 review round 8, sixth pre-review: escapes under nested masks, PEM } }); - it('keeps the private-key markers a structured summary masks whole', async () => { + it("reads a structured summary's members before the walk masks them whole", async () => { const begin = '-----BEGIN RSA PRIVATE KEY-----'; const end = '-----END RSA PRIVATE KEY-----'; - // A credential member or flag argument masked whole keeps a marker whose partner lies outside it, - // so a block that runs on into or out of it is still masked across members. + // The text rules read the result's JSON once before the walk's whole masks, which that same pass + // applies, so a private-key block that runs into or out of a credential member or flag argument is + // masked across members, whatever the masked value holds. for (const [result, summary, secret] of [ [ { private_key: `${begin}\nMIIEAAAAfirst`, rest: `MIIEBBBBsecondbodyline\n${end}` }, @@ -2665,12 +2666,12 @@ describe('PR95 review round 8, sixth pre-review: escapes under nested masks, PEM ], [ { a: `${begin}\nMIIEpAIBAAKCAQEAbodylineone`, private_key: `bodylinetwo\n${end}` }, - '{"a":"[redacted]"}', + '{"a":"[redacted]}', 'bodylineone', ], [ ['--private-key', begin, 'MIIEpAIBAAKCAQEAsecretbodyline1', 'abcdefsecretbodyline2', end], - '["--private-key","[redacted][redacted]"]', + '["--private-key","[redacted]""]', 'secretbodyline', ], [ @@ -2678,7 +2679,6 @@ describe('PR95 review round 8, sixth pre-review: escapes under nested masks, PEM '["--[redacted]"]', 'MIIE', ], - // A member that ends one block and starts another keeps both markers. [ { a: `${begin}\nAAAbody1`, @@ -2689,48 +2689,62 @@ describe('PR95 review round 8, sixth pre-review: escapes under nested masks, PEM '{"a":"[redacted]","d":"public"}', 'BBBbody2', ], - // The END marker it keeps is its first, the one that may close a block begun before it. [ { a: `${begin}\nAAAfirstbody`, private_key: `bbb\n${end}\n${begin}\nccc\n${end}`, z: 'public', }, - '{"a":"[redacted]","z":"public"}', + '{"a":"[redacted],"z":"public"}', 'AAAfirstbody', ], - // The BEGIN marker it keeps is its last, the one that may open a block that runs on after it. [ { private_key: `${begin}\nAAA\n${end}\n${begin}\nBBBhead`, rest: `CCCbody\n${end}` }, '{"[redacted]}', 'CCCbody', ], - // Members inside arrays and objects keep them too. [ { items: [{ private_key: `${begin}\nMIIEnested`, rest: `NESTEDbody\n${end}` }] }, '{"items":[{"[redacted]}]}', 'NESTEDbody', ], - ] as const) { - const bounded = await boundForModel(result, BIG, {}); - expect(bounded.summary, 'SECRETS_B8_CORE_SUMMARY_PEM').toBe(summary); - expect(bounded.summary.includes(secret), 'SECRETS_B8_CORE_SUMMARY_PEM').toBe(false); - } - // A block inside the member is masked with its markers, so a lone marker elsewhere stays text. - for (const [result, summary] of [ + // A credential member that holds a whole block of its own, an object, or a scheme label. [ - { a: `docs mention ${begin} here`, private_key: `${begin}\nMIIEwhole\n${end}` }, - `{"a":"docs mention ${begin} here","[redacted]}`, + { + stdout: `${begin}\nMIIEpAIBAAKCAQEAQZXSECRETBODYLINE1abc\n[output truncated]`, + private_key: `${begin}\nMIIBOwIBAAJBAKj34\n${end}`, + }, + '{"stdout":"[redacted]}', + 'QZXSECRETBODYLINE1', + ], + [ + { + credentials: { type: 'service_account', private_key: '-----BEGIN PRIVATE KEY-----' }, + body: 'MIIEvQQZXSECRET\n-----END PRIVATE KEY-----\n', + }, + '{"credentials":"[redacted]"}', + 'QZXSECRET', ], [ - { private_key: `${begin}\nMIIEwhole\n${end}`, b: `see ${end} there` }, - `{"[redacted],"b":"see ${end} there"}`, + { token: `Bearer x ${begin}`, rest: `MIIEvQQZXSECRET\n${end}` }, + '{"[redacted]}', + 'QZXSECRET', ], ] as const) { - expect((await boundForModel(result, BIG, {})).summary, 'SECRETS_B8_CORE_SUMMARY_PEM').toBe( - summary, - ); + const bounded = await boundForModel(result, BIG, {}); + expect(bounded.summary, 'SECRETS_B8_CORE_SUMMARY_PEM').toBe(summary); + expect(bounded.summary.includes(secret), 'SECRETS_B8_CORE_SUMMARY_PEM').toBe(false); } + expect( + ( + await boundForModel( + { private_key: `${begin}\nMIIEwhole\n${end}`, b: `see ${end} there` }, + BIG, + {}, + ) + ).summary, + 'SECRETS_B8_CORE_SUMMARY_PEM', + ).toBe(`{"[redacted],"b":"see ${end} there"}`); // The journal and the display projections keep their whole mask. expect( projectSecretShapedValue({ private_key: `${begin}\nMIIEAAAA`, rest: 'x' }).value, @@ -2812,3 +2826,176 @@ describe('PR95 review round 8, sixth pre-review: escapes under nested masks, PEM ).toBe('{"msg":"Use\\nToken handling"}'); }); }); + +describe('PR95 review round 9: closing views, unwalked members, wrapped markers and linear views', () => { + it('closes a quoted value in the views the key rule read it in before its rules merged', async () => { + // A value whose line break a scheme label's token takes closes as the next pass reads it. It is also + // read without the key rule's own masks, so another key's value that takes its closing quote, or ends + // at its opening one, does not keep it open; and where it is first read, with the scheme labels' + // tokens filled too, so a token that takes its line break lets it close at its real quote. + for (const [input, expected] of [ + ['password="correct horse battery Bearer\nxyzxyzxy token="q" more', '[redacted] more'], + ['token\\":"x password="Bearer\nabcdefghij Wsecret12 Wsecret34" tail', '[redacted] tail'], + ["password:\\t'x api_key='Bearer\nabcdefghij Wsecret12 Wsecret34' tail", '[redacted] tail'], + [ + 'api_key=password="password="Bearer\nabcdefghij Wsecret12 Wsecret34" tail', + '[redacted] tail', + ], + ['{"secret": "hunter2 QZXsecondsecret Bearer\npassword": "x"}', '{"[redacted]}'], + ['token\\\':"secret="\'QZXJ7WKM2V Bearer\nabcdefgh"', '[redacted]'], + ['accesskey\\"="token:"Token\rabcd1234\\QZXJ7WKM2V"', '[redacted]'], + [ + 'authorization="g://&a=m3z1VvzzvmzZZ@gcom/v1/a\\"\\QZXJ7WKM2V BASIC\nqv8x~V6t"', + '[redacted]', + ], + // A filled scheme token only moves a closing that a reading already reaches: a value that no + // reading closes is decided once the next key's value, which opens at its first quote, is known. + ['Authorization: "Bearer\nAuthorization: ""Zq7Wk3Xp9Rm2"', '[redacted]'], + ['password="Bearer\npassword=""Zq7Wk3Xp9Rm2"', '[redacted]'], + // So does one behind an opaque value whose mask ends with a backslash, where the scheme views are read. + [ + `x ?sig=${'a'.repeat(26)}\\" Authorization: "Bearer\nAuthorization: ""Zq7Wk3Xp9Rm2"`, + 'x ?sig=[redacted]" [redacted]', + ], + ] as const) { + expect(redactSecretShapedText(input), 'SECRETS_B9_CORE_CLOSING_VIEWS').toBe(expected); + expect(projectSecretShapedValue({ a: input }), 'SECRETS_B9_CORE_CLOSING_VIEWS').toEqual({ + value: { a: expected }, + settled: true, + }); + expect((await boundForModel(input, BIG, {})).summary, 'SECRETS_B9_CORE_CLOSING_VIEWS').toBe( + expected, + ); + } + }); + + it('masks a summary value through a later closer when one of its readings stops at a line break', async () => { + // One reading closes the value at a quote an opaque value's mask unescapes, the other stops at the + // line break before its real closer; the summary masks through that closer. + for (const input of [ + 'password="see ?sig=https://u:p@cdn.example/abc\\" Wsecret12\nmore"', + 'password="?sig=https://u:p@cdn.example/abc\\"Wq0zkx\n"', + "passwd='?g=ttps://u:p@h.example/\\n\\'PLANT9\n'", + ]) { + expect((await boundForModel(input, BIG, {})).summary, 'SECRETS_B9_CORE_SUMMARY_STOPPED').toBe( + '[redacted]', + ); + } + }); + + it("reads a structured summary's quoted value across a member masked whole", async () => { + for (const [result, summary, secret] of [ + [ + { argv: ['mysql', "--password='correct", 'horse', "battery'", '-h', 'db'] }, + '{"argv":["mysql","--[redacted]","-h","db"]}', + 'battery', + ], + [ + { argv: ['mysql', "--password='correcthorse", "battery'", '-h', 'db'] }, + '{"argv":["mysql","--[redacted]","-h","db"]}', + 'battery', + ], + [ + { note: 'x', pin: "password='abc", b: "Wdef12' tail" }, + '{"note":"x","pin":"[redacted]" tail"}', + 'Wdef12', + ], + ] as const) { + const bounded = await boundForModel(result, BIG, {}); + expect(bounded.summary, 'SECRETS_B9_CORE_SUMMARY_MEMBERS').toBe(summary); + expect(bounded.summary.includes(secret), 'SECRETS_B9_CORE_SUMMARY_MEMBERS').toBe(false); + } + }); + + it('reads a private-key marker that a line break, a tab or a doubled space splits in a summary', async () => { + for (const [input, summary] of [ + [ + 'PRIVATE_KEY=-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0QZXSECRET\n-----END PRIVATE\nKEY-----', + '[redacted]', + ], + [ + 'token=-----BEGIN RSA PRIVATE KEY-----\nMIIEbodyQZXSECRET\n-----END\nPRIVATE KEY-----', + '[redacted]', + ], + [ + 'secret=-----BEGIN\tPRIVATE KEY-----\nMIIEbodyQZXSECRET\n-----END PRIVATE KEY-----', + '[redacted]', + ], + [ + 'PRIVATE_KEY=-----BEGIN RSA PRIVATE KEY-----\nMIIEbodyQZXSECRET\n-----END RSA PRIVATE KEY-----', + '[redacted]', + ], + [ + '-----BEGIN PRIVATE\nKEY-----\nMIIEbodyQZXSECRET\n-----END PRIVATE KEY-----apikey:"', + '[redacted]', + ], + [ + 'pem: -----BEGIN RSA PRIVATE KEY-----\nMIIEbodyQZXSECRET\n-----END RSA PRIVATE KEY----- done', + 'pem: [redacted] done', + ], + ] as const) { + expect((await boundForModel(input, BIG, {})).summary, 'SECRETS_B9_CORE_WRAPPED_MARKERS').toBe( + summary, + ); + } + // The journal projection, which is compared rather than shown, reads such a marker as before. + const wrapped = + 'pem: -----BEGIN RSA PRIVATE KEY-----\nMIIEbody\n-----END RSA PRIVATE KEY----- done'; + expect(redactSecretShapedText(wrapped), 'SECRETS_B9_CORE_WRAPPED_MARKERS').toBe(wrapped); + }); + + it("keeps the walk's masks in place for a result whose own text holds a placeholder", async () => { + // The summary marks each value the walk masks whole with a private-use placeholder; a result that + // already holds one is summarized from the walk's masks alone, and its own text stays as it is. + const result = { password: 'Qz7secretvalue', note: 'x\uE0000\uE001y' }; + expect( + (await boundForModel(result, BIG, {})).summary, + 'SECRETS_B9_CORE_SUMMARY_PLACEHOLDER', + ).toBe('{"[redacted],"note":"x\uE0000\uE001y"}'); + }); + + it("reads an opaque parameter in a URL username behind that username's zone", () => { + expect( + redactSecretShapedText(`https://ab&tok=${'A'.repeat(26)}:pw@host.example/x?q=1`), + 'SECRETS_B9_CORE_OPAQUE_NAMES', + ).toBe('https://[redacted]@host.example/x?q=1'); + }); + + it( + 'keeps every projection linear where masks share a run of backslashes or end in one', + async () => { + // Each view of a quoted value is built once per pass, and a run of backslashes that many masks + // end in is read once. + for (const [input, summary] of [ + [`password="x\n${'token='.repeat(1_000_000)}${'\\'.repeat(1_000_000)} end`, '[redacted]'], + [`${'password="abcdefgh\n'.repeat(400_000)}token=abcdef\\ end`, '[redacted]'], + [ + `${'"password": "abc", '.repeat(400_000)}\\"https://h.example/x?sig=${'a'.repeat(30)}\\" end`, + '"[redacted], "[redacted],', + ], + ] as const) { + expect(redactSecretShapedText(input).endsWith(' end'), 'SECRETS_B9_CORE_LINEAR_VIEWS').toBe( + true, + ); + expect( + projectSecretShapedValue({ content: input }).settled, + 'SECRETS_B9_CORE_LINEAR_VIEWS', + ).toBe(true); + expect( + (await boundForModel(input, BIG, {})).summary.startsWith(summary), + 'SECRETS_B9_CORE_LINEAR_VIEWS', + ).toBe(true); + } + const structured = { + stdout: `password="first\nsecond" ${'log line ok\n'.repeat(600_000)}echo "token=abcdef12"\n`, + }; + expect( + (await boundForModel(structured, BIG, {})).summary.startsWith( + '{"stdout":"[redacted] log line ok', + ), + 'SECRETS_B9_CORE_LINEAR_VIEWS', + ).toBe(true); + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); +}); diff --git a/packages/core/src/tools/bounding.ts b/packages/core/src/tools/bounding.ts index 39dd2fc9..fe8f156d 100644 --- a/packages/core/src/tools/bounding.ts +++ b/packages/core/src/tools/bounding.ts @@ -538,18 +538,27 @@ function secretRanges(text: string, throughCloser: boolean): readonly SecretRang const record: SecretSpanVisitor = (start, end, replacement) => { ranges.push({ start, end, replacement }); }; + // The key rule's own masks, which a value that does not close is also read without. + const keyValues = new Set(); + const recordValue: SecretSpanVisitor = (start, end, replacement) => { + const range = { start, end, replacement }; + ranges.push(range); + keyValues.add(range); + }; + const schemes: SecretRange[] = []; visitSplitCredentials(text, report); visitUrlUserinfo(text, report); const names = fillRanges(text, owned); visitOpaqueQueryValues(names, text, report); - visitPemBlocks(text, report); + visitPemBlocks(text, report, throughCloser); // No value runs through a scheme label's token: `authorization: Bearer ` keeps both masks. - visitSchemeCredentials(text, record); + visitSchemeCredentials(text, (start, end, replacement) => { + record(start, end, replacement); + schemes.push({ start, end, replacement }); + }); const values = fillRanges(text, masked); - // The same masks with the backslashes each ends with kept, for a quote's second reading (`laterClosing`). - const kept = keepingEscapes(text, masked); - const escapes = kept === masked ? values : fillRanges(text, kept); - const unclosed = visitSecretKeyValues(names, values, escapes, record, throughCloser); + const views = quoteViews(text, masked, schemes); + const unclosed = visitSecretKeyValues(names, values, views, recordValue, throughCloser); // Well-known standalone credential shapes (OpenAI, Stripe, AWS incl. STS ASIA/ABIA, GitHub token + PAT, // GitLab PAT, Slack, Google API + OAuth, HuggingFace, npm, JWT). Kept as ONE alternation ON PURPOSE: a // single leftmost-longest scan covers two ADJACENT / EMBEDDED credential shapes (e.g. a JWT whose @@ -573,7 +582,7 @@ function secretRanges(text: string, throughCloser: boolean): readonly SecretRang } // A key's quoted value that does not close on its line is decided last, when every other range of the // pass is known (`visitUnclosedValues`). - if (unclosed.length > 0) visitUnclosedValues(text, values, ranges, unclosed, record); + if (unclosed.length > 0) visitUnclosedValues(text, values, ranges, keyValues, unclosed, record); // A summary also reads the line breaks and tabs that JSON escaping hides (`visitEscapedSeparators`). if (throughCloser) visitEscapedSeparators(text, record); return ranges; @@ -582,15 +591,31 @@ function secretRanges(text: string, throughCloser: boolean): readonly SecretRang /** * The ranges without the run of backslashes each ends with in `text`, or `ranges` itself when none ends * with one. Filled, they leave a quote right after a mask the escape it has in the text (`laterClosing`). + * Ranges that share a backslash run read it once: each run's start is remembered at every end it serves. */ function keepingEscapes(text: string, ranges: readonly SecretRange[]): readonly SecretRange[] { + const runStarts = new Map(); + const runStart = (end: number): number => { + let index = end; + while (text[index - 1] === '\\') { + const known = runStarts.get(index); + if (known !== undefined) { + index = known; + break; + } + index -= 1; + } + for (let position = end; position > index && text[position - 1] === '\\'; position -= 1) { + if (runStarts.has(position)) break; + runStarts.set(position, index); + } + return index; + }; let changed = false; const kept = ranges.map((range) => { - let end = range.end; - while (end > range.start && text[end - 1] === '\\') end -= 1; - if (end === range.end) return range; + if (text[range.end - 1] !== '\\') return range; changed = true; - return { ...range, end }; + return { ...range, end: Math.max(runStart(range.end), range.start) }; }); return changed ? kept : ranges; } @@ -599,18 +624,47 @@ function keepingEscapes(text: string, ranges: readonly SecretRange[]): readonly type QuotedValueEnd = ReturnType; /** - * Of a quoted value's two readings, the one that closes it later: one where the masks before or inside it - * are filled, as the next pass reads them, and one where each mask keeps the backslashes it ends with, as - * the text has them. A mask can take the backslash that escapes a later quote, which then closes the value - * early in the first reading (`password="Bearer\nabcdefgh token=xyz\"tail" more`), or the backslash before - * a quote that is the value's only closer (`password="say \"…?sig=<24 or more>\"`), which closes it in the - * first reading alone; the later closing of the two masks both. A reading that closes beats one that does - * not. + * Of two readings of a quoted value, the one that closes it later; a reading that closes beats one that + * does not. A quoted value is read in several views of its text (`quoteViews`, `visitUnclosedValues`): + * with the masks before or inside it filled, as the next pass reads them, and with each mask keeping the + * backslashes it ends with, as the text has them; and with the scheme labels' tokens filled too, as the + * key rule read them before this projection merged its rules. A mask can take the backslash that escapes a + * later quote, which then closes the value early in one reading + * (`password="Bearer\nabcdefgh token=xyz\"tail" more`), or the backslash before a quote that is the + * value's only closer (`password="say \"…?sig=<24 or more>\"`), which closes it in that reading alone; a + * scheme token can take the line break before the value's closer. The latest closing of all of them + * masks each of these. */ function laterClosing(first: QuotedValueEnd, second: QuotedValueEnd): QuotedValueEnd { return second.closed && (!first.closed || second.end > first.end) ? second : first; } +/** + * The views a quoted value is also read in beside `values` (`laterClosing`), when one of the earlier masks + * ends with a backslash: `plain`, which is `escapes`, those masks keeping the backslashes they end with; + * and `schemed`, `escapes` with the scheme labels' tokens filled too, when there is one, which only moves + * a closing that a plain reading reaches. With no such backslash every view reads like `values`, so none + * is built: a scheme token cannot move a closing that `values` itself reaches on its line. `escapes` also + * gives the summary's closers their second reading (`throughNearestCloser`). + */ +interface QuoteViews { + readonly escapes: string | undefined; + readonly plain: readonly string[]; + readonly schemed: readonly string[]; +} + +function quoteViews( + text: string, + masked: readonly SecretRange[], + schemes: readonly SecretRange[], +): QuoteViews { + const kept = keepingEscapes(text, masked); + if (kept === masked) return { escapes: undefined, plain: [], schemed: [] }; + const escapes = fillRanges(text, kept); + const schemed = schemes.length > 0 ? [fillRanges(text, [...kept, ...schemes])] : []; + return { escapes, plain: [escapes], schemed }; +} + /** * A summary's second reading, for the line breaks and tabs that JSON escaping hides: a structured result's * JSON writes each as two characters, so a credential or a scheme label right after one has no boundary, @@ -683,12 +737,18 @@ const PEM_BODY_MAX = 20_000; * a BEGIN marker and the first END marker that starts at most PEM_BODY_MAX characters after it, which * is what one regex with a lazy bounded body finds. That regex rescanned the whole bounded body after * every unterminated BEGIN marker; here one END search serves every BEGIN marker before the END it - * found, so the scan stays linear. + * found, so the scan stays linear. A summary also reads a marker whose words a line break, a tab or more + * than one space separates (`wrapped`), as a block wrapped at a fixed width can split one: its pass + * before normalization must not mask part of a marker that its one-line pass would pair. */ -function visitPemBlocks(text: string, mask: SecretSpanVisitor): void { - if (!text.includes('-----BEGIN ')) return; - const begin = /-----BEGIN [A-Z0-9 ]{0,40}PRIVATE KEY-----/g; - const end = /-----END [A-Z0-9 ]{0,40}PRIVATE KEY-----/g; +function visitPemBlocks(text: string, mask: SecretSpanVisitor, wrapped = false): void { + if (!text.includes('-----BEGIN')) return; + const begin = wrapped + ? /-----BEGIN\s[A-Z0-9\s]{0,40}PRIVATE\s{1,8}KEY-----/g + : /-----BEGIN [A-Z0-9 ]{0,40}PRIVATE KEY-----/g; + const end = wrapped + ? /-----END\s[A-Z0-9\s]{0,40}PRIVATE\s{1,8}KEY-----/g + : /-----END [A-Z0-9 ]{0,40}PRIVATE KEY-----/g; let found: RegExpExecArray | null | undefined; for (let marker = begin.exec(text); marker !== null; marker = begin.exec(text)) { const body = begin.lastIndex; @@ -790,22 +850,23 @@ function valueDelimiter(): RegExp { * string down), or an unquoted run of at least six characters up to whitespace, a quote, `,`, `;` or * `&`. A quoted value that does not close on its line, or one JSON string down before the end of the * string around it, is returned for the journal projection to decide once the pass's other ranges are - * known (`visitUnclosedValues`). The summary's projection (`throughCloser`) masks such a value through a - * later quote instead (`throughNearestCloser`). Keys are read in `keys`, which hides each URL - * userinfo's username (`UrlUserinfo.ownsUntil`): a key there names the user + * known (`visitUnclosedValues`). The summary's projection (`throughCloser`) masks a value that one of its + * readings stops at a line break through a later quote instead (`throughNearestCloser`), and through + * the closing of a reading that closes too. Keys are read in `keys`, which hides each URL userinfo's + * username (`UrlUserinfo.ownsUntil`): a key there names the user * (`https://x-access-token:@github.com/o/r`), and its value would run on through the public host and * path. Values are read in `values`, where every earlier mask is one run, as an applied mask would be, - * and a quoted value also in `escapes`, where each such mask keeps the backslashes it ends with; it closes - * at the later of the two (`laterClosing`). The key scan resumes at each value rather than after it, so a - * key inside a value is read too: `token=a>password: ` has a second value past the first one's - * end. One delimiter scan serves every unquoted value, and a quoted value is scanned no further than its - * line: a nested quoted value can only open at the closing quote of the one around it, and one that does - * not close leaves no quote of its kind on its line, so the scan stays linear. + * and a quoted value also in the other `views` (`quoteViews`); it closes at the latest of them + * (`laterClosing`). The key scan resumes at each value rather than after it, so a key inside a value is + * read too: `token=a>password: ` has a second value past the first one's end. One delimiter scan + * serves every unquoted value, and a quoted value is scanned no further than its line in each view: a + * nested quoted value can only open at the closing quote of the one around it, and one that does not + * close leaves no quote of its kind on its line, so the scan stays linear. */ function visitSecretKeyValues( keys: string, values: string, - escapes: string, + views: QuoteViews, report: SecretSpanVisitor, throughCloser: boolean, ): readonly UnclosedValue[] { @@ -827,14 +888,22 @@ function visitSecretKeyValues( const opening = escaped ? values[valueStart + 1] : values[valueStart]; if (opening === '"' || opening === "'") { const value = quotedValueEnd(values, valueStart); - const read = - escapes === values ? value : laterClosing(value, quotedValueEnd(escapes, valueStart)); + const readings = [value, ...views.plain.map((view) => quotedValueEnd(view, valueStart))]; + // A scheme label's token can take the line break before a closer that a reading reaches early; a + // value that no reading closes is decided once every range of the pass is known instead. + const first = readings.reduce(laterClosing); + const read = first.closed + ? views.schemed.map((view) => quotedValueEnd(view, valueStart)).reduce(laterClosing, first) + : first; const kind = escaped ? `\\${opening}` : opening; - if (read.closed) mask(match.index, read.end); - else if (!throughCloser) - unclosed.push({ start: match.index, quote: valueStart, end: read.end }); - else if (read.lineBreak) open.push({ start: match.index, end: read.end, quote: kind }); - else mask(match.index, read.end); + if (!throughCloser) { + if (read.closed) mask(match.index, read.end); + else unclosed.push({ start: match.index, quote: valueStart, end: read.end }); + continue; + } + const stopped = readings.find((reading) => !reading.closed && reading.lineBreak); + if (read.closed || stopped === undefined) mask(match.index, read.end); + if (stopped !== undefined) open.push({ start: match.index, end: stopped.end, quote: kind }); continue; } if (delimiterAt < valueStart) { @@ -843,7 +912,7 @@ function visitSecretKeyValues( } if (delimiterAt - valueStart >= 6) mask(match.index, delimiterAt); } - for (const range of throughNearestCloser(values, escapes, masked, open)) { + for (const range of throughNearestCloser(values, views.escapes, masked, open)) { report(range.start, range.end, '[redacted]'); } return unclosed; @@ -858,39 +927,47 @@ interface UnclosedValue { /** * The journal projection's masks for key values that do not close on their line (`visitSecretKeyValues`). - * Each is read again in `closing`, where every other range of the pass is filled: a mask that takes its - * line break (a scheme label's token, a nested key's value) joins its lines as the next pass would read - * them, so a value that then closes is masked whole, as it was before this rule; it closes at the later of - * that reading and the one where each range keeps the backslashes it ends with (`laterClosing`). - * Otherwise it is read as an unquoted run from after its quote, in `values`, no further than where it - * stopped: its quote may instead close a string the key sits in (`"token=" in prose`), and masking on - * past the run would give calls that differ only in that later text one projection. A run starts right - * after a quote, where the delimiter scan of every earlier run stops, so no two runs share a scan; a - * reading in `closing` stops at the next quote that could close it, so the readings of one kind of quote - * share none either. + * Each is read again where every other range of the pass is filled: a mask that takes its line break (a + * scheme label's token, a nested key's value) joins its lines as the next pass would read them, so a + * value that then closes is masked whole, as it was before this rule. It is read with and without the + * key rule's own masks (`keyValues`), since another key's value can take its closing quote or end at its + * opening one, and in each with every range keeping the backslashes it ends with too; it closes at the + * latest of these readings (`laterClosing`). A reading whose view fills the value's own quote is skipped: + * there another range decides that quote. Otherwise the value is read as an unquoted run from after its + * quote, in `values`, no further than where it stopped: its quote may instead close a string the key sits + * in (`"token=" in prose`), and masking on past the run would give calls that differ only in that later + * text one projection. A run starts right after a quote, where the delimiter scan of every earlier run + * stops, so no two runs share a scan; a reading stops at the next quote that could close it in its view, + * so the readings of one kind of quote share none either. */ function visitUnclosedValues( text: string, values: string, ranges: readonly SecretRange[], + keyValues: ReadonlySet, unclosed: readonly UnclosedValue[], report: SecretSpanVisitor, ): void { - const closing = fillRanges(text, ranges); - const kept = keepingEscapes(text, ranges); - const escapes = kept === ranges ? closing : fillRanges(text, kept); + const others = ranges.filter((range) => !keyValues.has(range)); + const views: string[] = []; + for (const list of others.length === ranges.length ? [ranges] : [ranges, others]) { + views.push(fillRanges(text, list)); + const kept = keepingEscapes(text, list); + if (kept !== list) views.push(fillRanges(text, kept)); + } const delimiter = valueDelimiter(); for (const value of unclosed) { const from = value.quote + (values[value.quote] === '\\' ? 2 : 1); - // The quote itself may be inside another range, which then decides it. - if (closing.slice(value.quote, from) === values.slice(value.quote, from)) { - const first = quotedValueEnd(closing, value.quote); - const reread = - escapes === closing ? first : laterClosing(first, quotedValueEnd(escapes, value.quote)); - if (reread.closed) { - report(value.start, reread.end, '[redacted]'); - continue; - } + const quote = values.slice(value.quote, from); + let reread: QuotedValueEnd | undefined; + for (const view of views) { + if (view.slice(value.quote, from) !== quote) continue; + const reading = quotedValueEnd(view, value.quote); + reread = reread === undefined ? reading : laterClosing(reread, reading); + } + if (reread?.closed === true) { + report(value.start, reread.end, '[redacted]'); + continue; } delimiter.lastIndex = from; const end = Math.min(delimiter.exec(values)?.index ?? values.length, value.end); @@ -947,21 +1024,22 @@ function closingKind(text: string, index: number): string | undefined { * not, and only else to the end of its line. Masked values that take every later quote leave what lies * between them uncertain (`token='\ntoken='''`), so the summary masks through all of it. Like a * value that closes, it is read twice, with every mask filled and with each mask keeping the backslashes - * it ends with (`escapes`), and masked through the later of the two (`laterClosing`). One backward scan, - * from the end of the text to the earliest end of an open value; each run of backslashes is read once, - * before the quote it escapes. + * it ends with (`escapes` for the earlier masks), and masked through the later of the two + * (`laterClosing`). One backward scan, from the end of the text to the earliest end of an open value; + * each run of backslashes is read once, before the quote it escapes. */ function throughNearestCloser( values: string, - escapes: string, + escapes: string | undefined, masked: readonly SecretRange[], open: readonly OpenValue[], ): readonly { readonly start: number; readonly end: number }[] { if (open.length === 0) return []; const byEnd = open.toSorted((left, right) => right.end - left.end); const filled = fillRanges(values, masked); - const kept = keepingEscapes(escapes, masked); - const keptFilled = kept === masked && escapes === values ? filled : fillRanges(escapes, kept); + const base = escapes ?? values; + const kept = keepingEscapes(base, masked); + const keptFilled = escapes === undefined && kept === masked ? undefined : fillRanges(base, kept); // Where the nearest quote of each kind outside the masked values ends, and where the last one does, in // each reading. const outside = new Map(); @@ -981,10 +1059,10 @@ function throughNearestCloser( if (kind !== undefined) outside.set(kind, index + 1); const raw = closingKind(values, index); if (raw !== undefined && !last.has(raw)) last.set(raw, index + 1); - if (keptFilled === filled) continue; + if (keptFilled === undefined) continue; const keptKind = closingKind(keptFilled, index); if (keptKind !== undefined) keptOutside.set(keptKind, index + 1); - const keptRaw = closingKind(escapes, index); + const keptRaw = closingKind(base, index); if (keptRaw !== undefined && !keptLast.has(keptRaw)) keptLast.set(keptRaw, index + 1); } return ranges; @@ -1100,43 +1178,82 @@ export function projectSecretShapedValue(value: unknown): { /** How the walk projects each string: the display projection, or the journal's settle-checking one. */ type TextProjection = (text: string) => string; -/** What the walk puts in place of a value it masks whole: a credential member or a credential flag's argument. */ -type WholeMask = (value: unknown) => string; +/** + * What the walk puts in place of a value it masks whole, a credential member or a credential flag's + * argument, given that value and its `mask` (`[redacted]`, or a canonical scheme label and `[redacted]`). + */ +type WholeMask = (value: unknown, mask: string) => string; -/** The journal's and the display's whole mask. */ -const maskWhole: WholeMask = () => '[redacted]'; +/** The journal's and the display's whole mask: the mask itself. */ +const maskWhole: WholeMask = (_value, mask) => mask; /** - * The summary walk's whole mask (`toText`). It keeps a private-key END marker that may close a block opened - * before the value and a BEGIN marker that may open one continuing after it, so the summary's text rules - * still pair each with a marker in another member and mask the key's body there. A marker names a key - * type, not key material; a block whose markers are both inside the value is masked with it. + * The text a non-string result's summary is made from (`toText`): its JSON with every value the walk masks + * whole (`redactSecretShapedWalk`) masked, and read once by the text rules before those masks, so a quoted + * value or a PEM block that runs into or out of such a value keeps the context it is read in, as the JSON + * of the unwalked result would give it. The walk leaves a private-use placeholder for each such value; the + * value's own JSON goes back where its placeholder is, and its mask is one more range of that reading. A + * result whose JSON already holds a placeholder character, or one that has no JSON form, keeps the walk's + * masks in place and skips the reading. */ -function maskWholeKeepingPemMarkers(value: unknown): string { - if (typeof value !== 'string' || !value.includes('-----')) return '[redacted]'; - const begin = /-----BEGIN [A-Z0-9 ]{0,40}PRIVATE KEY-----/g; - const end = /-----END [A-Z0-9 ]{0,40}PRIVATE KEY-----/g; - let firstBegin: RegExpExecArray | undefined; - let lastBegin: RegExpExecArray | undefined; - for (let marker = begin.exec(value); marker !== null; marker = begin.exec(value)) { - firstBegin ??= marker; - lastBegin = marker; +function structuredSummaryText(safe: unknown): string { + const holes: { readonly value: unknown; readonly mask: string }[] = []; + const marked = jsonText(() => + redactSecretShapedWalk( + safe, + new WeakSet(), + (text) => text, + undefined, + (value, mask) => `\uE000${holes.push({ value, mask }) - 1}\uE001`, + ), + ); + if (holes.length === 0) return marked; + const placeholders = [...marked.matchAll(/\uE000(\d+)\uE001/g)]; + if (placeholders.length !== holes.length || marked.split('\uE000').length - 1 !== holes.length) { + return jsonText(() => redactSecretShapedWalk(safe, new WeakSet(), (text) => text)); + } + const parts: string[] = []; + const masks: SecretRange[] = []; + let length = 0; + let copied = 0; + for (const placeholder of placeholders) { + const hole = holes[Number(placeholder[1])]; + if (hole === undefined) continue; + // A placeholder that is a whole JSON string stands for any value; one inside a string (a + // `--password=` argument) stands for a string's text there. + let start = placeholder.index; + let end = start + placeholder[0].length; + let json: string; + let mask: string; + if (marked[start - 1] === '"' && marked[end] === '"') { + start -= 1; + end += 1; + // A value with no JSON form (a function, a bigint, a cycle) is read as its mask. + json = jsonValue(hole.value) ?? JSON.stringify(hole.mask); + mask = JSON.stringify(hole.mask); + } else { + json = JSON.stringify(typeof hole.value === 'string' ? hole.value : hole.mask).slice(1, -1); + mask = JSON.stringify(hole.mask).slice(1, -1); + } + const before = marked.slice(copied, start); + parts.push(before, json); + length += before.length; + masks.push({ start: length, end: length + json.length, replacement: mask }); + length += json.length; + copied = end; } - let firstEnd: RegExpExecArray | undefined; - let lastEnd: RegExpExecArray | undefined; - for (let marker = end.exec(value); marker !== null; marker = end.exec(value)) { - firstEnd ??= marker; - lastEnd = marker; + parts.push(marked.slice(copied)); + const text = parts.join(''); + return replaceRanges(text, [...secretRanges(text, true), ...masks]); +} + +/** The JSON of one value, or `undefined` where it has none or its serialization throws. */ +function jsonValue(value: unknown): string | undefined { + try { + return JSON.stringify(value); + } catch { + return undefined; } - const closing = - firstEnd !== undefined && (firstBegin === undefined || firstBegin.index > firstEnd.index) - ? firstEnd[0] - : ''; - const opening = - lastBegin !== undefined && (lastEnd === undefined || lastEnd.index < lastBegin.index) - ? lastBegin[0] - : ''; - return `${closing}[redacted]${opening}`; } /** @@ -1255,7 +1372,7 @@ function redactCredentialArgument(flag: string, value: unknown, whole: WholeMask if (flag.toLowerCase() === '-p' && typeof value === 'string' && looksLikePort(value)) { return value; } - return whole(value); + return whole(value, '[redacted]'); } /** @@ -1311,14 +1428,14 @@ function redactUnderKey( const scheme = /^(bearer|basic|token)[ \t]+\S/i.exec(item)?.[1]?.toLowerCase(); switch (scheme) { case 'bearer': - return 'Bearer [redacted]'; + return whole(item, 'Bearer [redacted]'); case 'basic': - return 'Basic [redacted]'; + return whole(item, 'Basic [redacted]'); case 'token': - return 'Token [redacted]'; + return whole(item, 'Token [redacted]'); } } - return whole(item); + return whole(item, '[redacted]'); } function redactSecretShapedWalk( @@ -1362,9 +1479,9 @@ function redactSecretShapedWalk( /** * Render any result to the text the model would see (a string is itself; else compact JSON) and to the text * its summary is made from. Inline media bytes are redacted first so the summary/spill/preview can never - * carry base64 (I3). A non-string result's summary text is its JSON once the walk has masked every member - * under a credential-named key and every argument after a credential flag whole (`redactSecretShapedWalk`), - * which no text rule reads. The text rules then read the JSON as one text (`makeSummary`), a value one JSON + * carry base64 (I3). A non-string result's summary text is its JSON with every member under a + * credential-named key and every argument after a credential flag masked whole (`structuredSummaryText`), + * which no text rule reads. The text rules read the JSON as one text (`makeSummary`), a value one JSON * string down included, so a PEM block or a quoted value that runs across members keeps the context it is * read in; projecting each string alone would mask a key or a PEM marker that a later member's text needs. */ @@ -1387,18 +1504,7 @@ function toText(result: unknown): { readonly text: string; readonly summaryText: } catch { return { text: '[unserializable]', summaryText: '[unserializable]' }; // throwing getter during redaction } - return { - text: jsonText(() => safe), - summaryText: jsonText(() => - redactSecretShapedWalk( - safe, - new WeakSet(), - (text) => text, - undefined, - maskWholeKeepingPemMarkers, - ), - ), - }; + return { text: jsonText(() => safe), summaryText: structuredSummaryText(safe) }; } /** diff --git a/tools/causal-controls/provenance.test.mjs b/tools/causal-controls/provenance.test.mjs index 8a082124..06452041 100644 --- a/tools/causal-controls/provenance.test.mjs +++ b/tools/causal-controls/provenance.test.mjs @@ -355,10 +355,10 @@ test('secrets workload corrections retain independent complete-count and reader- 'core-summary-whole-json', 'core-journal-key-collision', 'core-escaped-value-start', - // Sixth pre-review: a summary's escaped separators and the PEM markers its walk keeps. + // Sixth pre-review and round 9: a summary's escaped separators and its unwalked reading. 'core-summary-escape-view', 'core-summary-pem-markers', - ].map((id) => [id, ['packages/core/src/tools/bounding.test.ts'], 210]), + ].map((id) => [id, ['packages/core/src/tools/bounding.test.ts'], 217]), ['core-unsettled-refusal', ['packages/core/src/tools/registry.test.ts'], 115], ...[ 'diagnostic-joined-before-masking', diff --git a/tools/secrets-controls/README.md b/tools/secrets-controls/README.md index 1b312873..96898776 100644 --- a/tools/secrets-controls/README.md +++ b/tools/secrets-controls/README.md @@ -178,7 +178,7 @@ modern `sk-` body characters, joined-span application, the original-text left-bo check, split scheme labels, split API-header labels and raw `?`/`#` password recognition independently. Four LLM controls remove the before-masking LF/TAB probe, its original left-boundary check, split API-header recognition and raw-password recognition. -Every core projection phase executes the complete 210-case bounding file (the journal-identity +Every core projection phase executes the complete 217-case bounding file (the journal-identity control's phases the complete 115-case registry file); every new LLM phase executes the complete 224-case scrubber file. Independent ordinary tooling tests pin these inventories, the 278-case overlap cohort, 180-case CLI diagnostic cohort and 18-case command-export cohort. Actual collection still has to match in every phase. @@ -261,7 +261,7 @@ previous-word prefilter and the rule's own boundary predicate; retaining either would leave the tested policy active. Header/key recovery removals still change their original-rule boundary only, and all removals retain the same canonical matchers and bounded raw/joined standalone visitors. Current collection is 26 -complete files / 1,640 cases, with each control's exact workload and named target +complete files / 1,647 cases, with each control's exact workload and named target checked separately. These are updated declarations and author checks, not new physical causal captures or a clean review round. R3 is preserved as NONCLEAN; later corrective acceptance belongs to the owning phase register. Prior selected @@ -270,8 +270,8 @@ later corrective acceptance belongs to the owning phase register. Prior selected The current 130-control / 390-phase full declaration runs as two round-robin shards of 65 controls (`--shard 1/2` and `--shard 2/2`), each exporting its own evidence: each projects 1,177 retained files (65 × 18 + 7) of the unchanged 2,000. Locally both exports held exactly that many and measured -49.83 MiB and 49.76 MiB, final export record included, against the unchanged -64-MiB bound. That is one local measurement at the round-8 source, not a CI capture; one export of +50.19 MiB and 50.09 MiB, final export record included, against the unchanged +64-MiB bound. That is one local measurement at the round-9 source, not a CI capture; one export of the whole family would exceed the byte bound. Historical 41-control captures retain their original source and cohort attribution; selected B2 or original-boundary recovery captures are not full-family acceptance. @@ -507,6 +507,39 @@ it. Four mutations are output-neutral and have neither a test nor a control: the only skip work, and preferring a second reading that closes at all, since a reading that keeps the masks' backslashes closes only at a quote where the other reading has closed by then. +The ninth round found two high findings in round 8's own code and eight medium regressions against +base in five shapes. Round 8 compared whole views with `===` inside its loops and walked a run of +backslashes once per mask that ends in it, both quadratic at admitted size: each view of a quoted +value is now built once per pass, its presence a value decided once, and a run's start is remembered +at every end it serves. A value whose line break a scheme label's token takes did not close where +another key's value took its closing quote or ended at its opening one, and a reading closed early +where a filled scheme token would have let it reach its real quote: such a value is now reread with +and without the key rule's own masks, and the key rule's first decision also reads a view with the +masks' backslashes kept and the scheme tokens filled, which only moves a closing that a plain +reading reaches. A summary took one reading's early closing over another reading that stops at a +line break, and now masks such a value through its later closer. A structured summary's walk +replaced credential members and flag arguments before the text rules read the JSON: the summary now +reads the unwalked JSON once, each masked value spliced back where the walk left a private-use +placeholder, with the walk's masks as ranges of that pass, and round 8's kept PEM markers are +withdrawn. A summary's pass before normalization cut a private-key marker that a line break, a tab +or more than one space splits, and a summary now reads such markers. `core-summary-pem-markers`, +`core-summary-whole-json` and `core-unclosed-closing-view` are re-anchored, and +`core-key-value-through-masks` now targets round 7's reading through earlier masks, since the +unclosed value's reread also reads round 6's quoted witnesses; no control is added, and a control +for the later closing is deferred. Pinned by tests without a control, each killed by its +mutation: the quoted value's plain and scheme views, the gate that keeps a scheme view to a closing +a plain reading reaches, and the latest closing; the summary's stopped reading and its later closer; +the unclosed value's views with and without the key rule's masks, their kept variants, the guard on +a view that fills the value's quote and the latest reread; the unwalked reading's placeholders, +whole and inside a string, a value with no JSON form, its mask and its collision guard; the +scheme-labelled and argument whole masks; the wrapped markers in the summary alone, their whitespace +runs and the prefilter; and the opaque parameter in a URL username. An admitted-size case deadline +pins the four quadratic shapes in the journal, display and summary paths; a timeout counts as +infrastructure, so no control can certify it. Five mutations are output-neutral and have neither a +test nor a control: the closing mask of a summary value that also stops at a line break, which its +later closer covers; a kept run's clamp to its range, whose empty ranges a fill skips anyway; and +three shortcuts that keep or reuse a view when no mask ends with a backslash. + The thin entry point uses the same coordinator, outcome and bounded process join as the filesystem family in [causal-controls](../causal-controls/README.md). The provider-header diagnostic control pins the store's fixed throwable message diff --git a/tools/secrets-controls/mutations.mjs b/tools/secrets-controls/mutations.mjs index 424c48f3..41bf273c 100644 --- a/tools/secrets-controls/mutations.mjs +++ b/tools/secrets-controls/mutations.mjs @@ -334,7 +334,7 @@ const withoutCoreOriginalBoundary = coreOriginalBoundary ) .replace(" if (boundary?.test(text[originalStart - 1] ?? '')) return;", ''); /** The complete core projection test file every core projection control runs. */ -const CORE_PROJECTION_CASES = 210; +const CORE_PROJECTION_CASES = 217; function coreProjectionControl(id, before, after, targets, marker = coreProjectionMarker) { return control( id, @@ -1491,7 +1491,7 @@ export const mutations = [ 'core-key-value-through-masks', ' const values = fillRanges(text, masked);', ' const values = names;', - ['reads a value through the masks before it, line breaks included'], + ['reads a value through earlier masks of every kind, in text order'], 'SECRETS_B6_CORE_VALUE_THROUGH_MASK', ), coreProjectionControl( @@ -1616,8 +1616,8 @@ export const mutations = [ ), coreProjectionControl( 'core-unclosed-closing-view', - ' const closing = fillRanges(text, ranges);', - ' const closing = values;', + ' views.push(fillRanges(text, list));', + ' views.push(values);', ['reads a quoted value that does not close on its line as an unquoted run'], 'SECRETS_B8_CORE_UNCLOSED_QUOTE', ), @@ -1634,8 +1634,8 @@ export const mutations = [ ), coreProjectionControl( 'core-summary-whole-json', - ' (text) => text,', - ' redactSecretShapedSummaryText,', + ' (text) => text,', + ' redactSecretShapedSummaryText,', ["reads a structured result's values across its line breaks and members"], 'SECRETS_B8_CORE_SUMMARY_JSON', ), @@ -1660,8 +1660,8 @@ export const mutations = [ ['reads a label at a line start inside a split continuation'], 'SECRETS_B8_CORE_CONTINUATION_LABEL', ), - // Sixth pre-review: a summary reads the line breaks and tabs that JSON escaping hides, and its walk - // keeps the private-key markers a whole mask would take from a block that crosses members. + // Sixth pre-review and round 9: a summary reads the line breaks and tabs that JSON escaping hides, and + // reads a structured result's JSON before the walk's whole masks, which that pass applies. coreProjectionControl( 'core-summary-escape-view', ' if (throughCloser) visitEscapedSeparators(text, record);', @@ -1671,9 +1671,9 @@ export const mutations = [ ), coreProjectionControl( 'core-summary-pem-markers', - ' maskWholeKeepingPemMarkers,', - ' maskWhole,', - ['keeps the private-key markers a structured summary masks whole'], + ' return replaceRanges(text, [...secretRanges(text, true), ...masks]);', + ' return replaceRanges(text, masks);', + ["reads a structured summary's members before the walk masks them whole"], 'SECRETS_B8_CORE_SUMMARY_PEM', ), coreProjectionControl( From 9eb3815633363a9344b4e5cfc1a134697f052eba Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Thu, 22 Oct 2026 02:28:24 +0300 Subject: [PATCH 19/97] fix(core): close the PR95 increment after its tenth review round MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 10 of the PR95 increment's review (at 59c64498), the last under the maintainer's 2026-10-21 round cap, reported eleven verified findings: seven medium and four low as verified (three of the mediums are one finding), no high and no blocker. On 2026-10-22 the maintainer counted GEN-R10-1 as high and approved the two private-key block gaps found after round 9 as deferrals; every other finding is recorded in deferred-tasks.md, as the round cap provides. Core projection (packages/core/src/tools/bounding.ts): - In a structured result's JSON a query value at the end of a line ran on through the next line's key and took the backslash of its value's escaped quote, so a `cat .env` result's POSTGRES_PASSWORD="…" after `?sslmode=require` showed in the summary that agent:tool_result carries (GEN-R10-1, unchanged since base). A summary now opens a key's value whose escaped opening quote ends an earlier mask's backslashes where that mask keeps them, reading it only in the views that keep them. The journal keeps its reading, so v5 digests do not change; a test pins that. - The docblocks that counted one summary difference, or said that no text rule reads a whole-masked member, are corrected (DOC-R10-2). Tests and controls: - A test with five witnesses (run_command, MCP text part, CRLF, Mongo and a tab) and a journal pin; 218 bounding cases. A mutation run of six mutants on the new code kills four; the two survivors are output-neutral, since a mask the kept view keeps ends before a delimiter, where a view that keeps the backslashes closes no later than the filled one. - core-quoted-value-scan and core-escaped-quote are re-anchored on the changed lines; the family stays 130 controls / 513 targets and witnesses in two shards of 65, whose local exports held 1,177 files each and measured 50.24 MiB and 50.14 MiB. - Differentials against 59c64498 and base: the journal and display text identical on 250,000 inputs; every summary that differs on 320,000 inputs shows a subsequence of 59c64498's visible text; no planted-token regression in the generators or in an exhaustive run of 837,606 sequences. Docs: built-in-tools.md (the kept opening and its journal limit; the wrapped markers' whitespace bound, the U+E000 placeholder's fallback and a PEM END marker glued to another match, named where the document claimed more), effect-journal.md §11 (the closing readings of a value that closes, INT-R10-2), deferred-tasks.md (round 10's medium and low findings with examples and re-checks, the journal half of GEN-R10-1, the two approved private-key block entries, the PR95 items of the PR #95 review closed), the phase register (round 10, the rulings, the closure and the round-9 records that DOC-R10-1 found stale), the secrets controls README and the status lines. The increment closes under the round cap. It is not independently accepted in the sense of the approved bar: round 10 was NONCLEAN, and this correction has author verification only. Refs: ADR-0113 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 9 +- CLAUDE.md | 9 +- docs/reference/shared-core/built-in-tools.md | 45 +++++--- docs/reference/shared-core/effect-journal.md | 11 +- docs/roadmap/current.md | 15 ++- docs/roadmap/deferred-tasks.md | 107 +++++++++++++----- .../phase-2.5.5-hardening-and-remediation.md | 52 ++++++--- packages/core/src/tools/bounding.test.ts | 52 +++++++++ packages/core/src/tools/bounding.ts | 45 +++++--- tools/causal-controls/provenance.test.mjs | 2 +- tools/secrets-controls/README.md | 29 +++-- tools/secrets-controls/mutations.mjs | 11 +- 12 files changed, 283 insertions(+), 104 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c2f61143..5272df09 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -125,10 +125,11 @@ fixes on `development`, not yet on `main`); CI for it and the merge commit finis gate (red since at least PR #90) was cleared on 2026-10-19. A pre-implementation review of Steps 9–19 and the maintainer's [2026-10-19 rulings](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) give every open PR #95 finding an owner and record 29 increments (no new ADR). The PR95 increment -(the scrubber residuals, including both security residuals) is implemented on `development` and under -review: rounds 1–9 were NONCLEAN and are corrected ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)). Under the maintainer's -2026-10-21 masker acceptance bar and round cap, round 10 is the last: after it only high findings are -corrected, and its other findings go to deferred-tasks.md. Then Step 9a.1 (release map/custody substrate, #46/ADR-0111). `development` is pushed after each accepted increment. No provider, +(the scrubber residuals, including both security residuals) is closed on `development` under the +maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were +NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last +round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, +by the author alone). The other findings are recorded in deferred-tasks.md. Next is Step 9a.1 (release map/custody substrate, #46/ADR-0111). `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. See [docs/roadmap/current.md](docs/roadmap/current.md) for live status. diff --git a/CLAUDE.md b/CLAUDE.md index 2cb00ca6..0fa4df88 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -214,10 +214,11 @@ fixes on `development`, not yet on `main`); CI for it and the merge commit finis gate (red since at least PR #90) was cleared on 2026-10-19. A pre-implementation review of Steps 9–19 and the maintainer's [2026-10-19 rulings](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) give every open PR #95 finding an owner and record 29 increments (no new ADR). The PR95 increment -(the scrubber residuals, including both security residuals) is implemented on `development` and under -review: rounds 1–9 were NONCLEAN and are corrected ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)). Under the maintainer's -2026-10-21 masker acceptance bar and round cap, round 10 is the last: after it only high findings are -corrected, and its other findings go to deferred-tasks.md. Then Step 9a.1 (release map/custody substrate, #46/ADR-0111). `development` is pushed after each accepted increment. No provider, +(the scrubber residuals, including both security residuals) is closed on `development` under the +maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were +NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last +round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, +by the author alone). The other findings are recorded in deferred-tasks.md. Next is Step 9a.1 (release map/custody substrate, #46/ADR-0111). `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. For live status, per-PR history, milestone dates, and open obligations, see the canonical diff --git a/docs/reference/shared-core/built-in-tools.md b/docs/reference/shared-core/built-in-tools.md index 16d4016c..e0b6f2d0 100644 --- a/docs/reference/shared-core/built-in-tools.md +++ b/docs/reference/shared-core/built-in-tools.md @@ -207,27 +207,42 @@ the admitted text, and none uses a backtracking pattern whose stack an admitted quoted value is read by a character scan no further than its line, and a summary decides its values that stop at a line break by one backward scan. Ordinary spaces and punctuation separate public prose; the masker does not discard the whole diagnostic because a credential appears. A summary uses -the projection with one difference, since a summary is shown, never compared: a quoted value that one -of its readings stops at a line break is masked through the nearest later quote that could close it -outside the values the rule masks or, when every such quote is inside one, through the last of them, -and only else to the end of its line, in whichever reading reaches further, and through the closing -of a reading that closes too; one that reaches the end of the string around it or of the text without -closing is masked to that end. A structured (non-string) result is summarized from its JSON with +the projection with the differences below, since a summary is shown, never compared. A quoted value +that one of its readings stops at a line break is masked through the nearest later quote that could +close it outside the values the rule masks or, when every such quote is inside one, through the last +of them, and only else to the end of its line, in whichever reading reaches further, and through the +closing of a reading that closes too; one that reaches the end of the string around it or of the text +without closing is masked to that end. A key's value whose escaped opening quote ends an earlier +mask's backslashes opens where that mask keeps them, so a query value at the end of a line that runs +on through the next line's key in a structured result's JSON does not hide that key's value (a +`cat .env` result's `?sslmode=require\nPOSTGRES_PASSWORD=\"…\"` → `?sslmode=[redacted]\n`); the +journal projection keeps its reading, so the same JSON as a text argument still shows such a value, a +documented limit. A structured (non-string) result is summarized from its JSON with every member under a credential-named key and every argument after a credential flag masked whole (a count such as `input_tokens` too, the text rules then masking the member's name with it). The text rules read that JSON once before those masks, which the same pass applies, so a quoted value or a PEM block that runs into or out of a masked member is masked across members, as reading the unwalked JSON -masks it; they read the JSON as one text, a value one JSON string down included, so a JSON-escaped -line break is no delimiter and a PEM block or a quoted value that runs across members keeps its key -and markers. A summary also reads a private-key marker whose words a line break, a tab or more than +masks it. The summary marks those values with the private-use character U+E000, so a result whose JSON +already holds one is summarized from the walk's masks alone and such a value or block is not masked +across members there, a documented limit. The text rules read the JSON as one text, a value one +JSON string down included, so a JSON-escaped line break is no delimiter and a PEM block or a quoted +value that runs across members keeps its key and markers. A summary also reads a private-key marker whose words a line break, a tab or more than one space separates (`-----END RSA PRIVATE\nKEY-----`), as a block wrapped at a fixed width can split -one; the journal projection reads such a marker as before, and a structured result's summary does not -read one whose line break or tab its JSON escapes (`{"a":"…-----END RSA PRIVATE\nKEY-----"}`), a -documented limit. A private-key block is masked only from a BEGIN marker to an END marker that starts +one: one to eight whitespace characters between `PRIVATE` and `KEY`, and up to 40 characters of type +words and whitespace after `BEGIN` or `END`. A marker wrapped further is read only after the summary's +one-line normalization, which a key's value that takes the BEGIN marker first prevents +(`PRIVATE_KEY=-----BEGIN PRIVATE KEY-----\n\n-----END PRIVATE\n KEY-----` shows the body), +a documented limit. The journal projection reads such a marker as before, and a structured result's +summary does not read one whose line break or tab its JSON escapes +(`{"a":"…-----END RSA PRIVATE\nKEY-----"}`), a documented limit. A private-key block is masked only from a BEGIN marker to an END marker that starts at most 20,000 characters after it, so a block cut off before its END marker (`head -n 5 key.pem`) -stays visible in every projection, a documented limit. In a summary, a key's unquoted value behind -an escaped line break and indentation (`password:\n ` in a structured result's JSON) is not read, -a documented limit. Every summary also reads its text a second time with each +stays visible in every projection, a documented limit. A block ends at its first END marker even where +another rule's match is glued to it (`?sig=-----END …`): text after that marker stays, and a key's +quoted value whose closing quote lies inside the block shows its front +(`api_key='x -----BEGIN …' ?sig=-----END …` → `api_key='x [redacted]`); base masked these only +because its rules ran in turn and the glued match erased the marker, a documented relaxation. In a +summary, a key's unquoted value behind an escaped line break and indentation (`password:\n ` in a +structured result's JSON) is not read, a documented limit. Every summary also reads its text a second time with each escaped line feed, carriage return or tab (`\n`, `\r`, `\t`, behind any run of backslashes) as that separator, for the rules a separator decides: a credential or a scheme label right after one, and a credential split by one, are masked as in raw text (`{"stdout":"keys:\nsk-…"}` → diff --git a/docs/reference/shared-core/effect-journal.md b/docs/reference/shared-core/effect-journal.md index b70284ea..da05fcfd 100644 --- a/docs/reference/shared-core/effect-journal.md +++ b/docs/reference/shared-core/effect-journal.md @@ -391,10 +391,13 @@ of every rule before replacing anything, so a userinfo is masked together with t of the keys and query parameters in it but its username, which never reaches past an LF or TAB, and no mask removes another credential's context; reads a key or scheme label inside another value as a credential of its own, and a value through earlier masks; masks userinfo split by an LF or TAB at -either edge or inside its `scheme://`; reads a quoted value one JSON string down too, and one that -does not close on its line, or before the end of its string, as an unquoted run that stops there — -unless another mask of the pass takes its line break and it closes, as the next pass read it before -(at the latest of its readings, with and without the key rule's own masks and the masks' trailing backslashes) — so the text after a quote that may close a string around the key keeps calls distinct; reads a +either edge or inside its `scheme://`; reads a quoted value one JSON string down too, and closes a +quoted value at the latest of its readings, with earlier masks keeping their trailing backslashes and, +once a reading closes, with the scheme labels' tokens filled; reads one that does not close on its +line, or before the end of its string, as an unquoted run that stops there — unless another mask of +the pass takes its line break and it closes, as the next pass read it before (at the latest of its +readings, with and without the key rule's own masks and the masks' trailing backslashes) — so the +text after a quote that may close a string around the key keeps calls distinct; reads a split header label or a scheme label right after a credential or a PEM block, and a header label at a line start inside a split continuation; and repeats the projection until it is stable. These change a `v5:` projection only for the arguments they project differently, such as userinfo diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index 380be2df..b0a9a7e3 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -1,7 +1,7 @@ # Current state > Status: Living -> Last updated: 2026-10-21 +> Last updated: 2026-10-22 - **Related**: [README.md](README.md), [phases/phase-2.5-cli-consolidation.md](phases/phase-2.5-cli-consolidation.md), [phases/phase-2.5.5-hardening-and-remediation.md](phases/phase-2.5.5-hardening-and-remediation.md), [phases/phase-2-cli.md](phases/phase-2-cli.md), [deferred-tasks.md](deferred-tasks.md), [../project-structure.md](../project-structure.md), [../tech-stack.md](../tech-stack.md) @@ -164,11 +164,14 @@ security residuals that predate the follow-up — an `api-key:Bearer ` to Wave 3 implementation continues on `development`: Steps 1–8 are merged. A pre-implementation document/source review of Steps 9–19 and the maintainer's rulings are recorded in the [2026-10-19 rulings](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) -(no new ADR; 29 recorded increments). **The PR95 increment is implemented on `development` and under review** — the PR #95 -scrubber residuals, including the two security residuals; rounds 1–9 were NONCLEAN and are corrected, as its -[implementation checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20) records. Under the maintainer's 2026-10-21 masker acceptance bar and round cap, round 10 is the last: after it -only its high findings are corrected, and its other findings are recorded in -[deferred-tasks.md](deferred-tasks.md#pr-95-masker-low-edge-cases--approved-deferrals-2026-10-21). **Step 9a.1** follows, the first increment of the release map/custody substrate (#46, ADR-0111), whose +(no new ADR; 29 recorded increments). **The PR95 increment is closed on `development`** — the PR #95 +scrubber residuals, including the two security residuals — under the maintainer's 2026-10-21 masker acceptance +bar and round cap, as its +[implementation checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20) records: rounds 1–10 were NONCLEAN; the high and +medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the +maintainer counted as high on 2026-10-22, is corrected (in the summary, by the author alone). Its other findings +are recorded in +[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1** follows, the first increment of the release map/custody substrate (#46, ADR-0111), whose 9a.1–9d increments carry a dedicated archive/crypto security sitting. Steps 10–19 follow the [increment table](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) and end with whole-wave acceptance at Step 19. `development` is pushed after each accepted increment; PR #95 no longer bounds the diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index 4c47b6ac..f0c6b1ef 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -2,7 +2,7 @@ > Status: Living -> Last updated: 2026-10-21 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Steps 9–19 not started; the PR95 increment is under review on `development`). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's low edge cases with the maintainer's 2026-10-21 acceptance bar. Existing approved deferrals remain. +> Last updated: 2026-10-22 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Steps 9–19 not started; the PR95 increment is closed on `development` under the maintainer's 2026-10-21 round cap). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's edge cases with the maintainer's 2026-10-21 acceptance bar and round cap and the 2026-10-22 rulings. Existing approved deferrals remain. > The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) records completed prerequisites; full M/N lifecycle and other scheduled work remain with their owners. > > Earlier triage: the Phase-2.6 rewrite triaged every open item, the 2026-07-19 full-project @@ -393,12 +393,12 @@ measured or counted here. `ci.yml` and the Sonar backlog section below), the S4036 accepted as on the PR #95 branch. SonarCloud then reported zero open vulnerabilities and an OK gate for `main`. No rule, threshold or gate was changed; the per-issue record is local and Git-ignored. *(was medium · process)* -- [ ] **Scheduled → Wave 3 PR95. An `api-key` header label erases the `Bearer`/`Basic` label that follows it, so the token leaks.** *Low (security).* *(author)* `scrubSecrets('api-key:Bearer abcdefghijkl123456')` returns `api-key:[REDACTED] abcdefghijkl123456` — the `api-key` rule consumes the word `Bearer` as the header's value and the Bearer rule then has no label to anchor on. The pre-PR matcher returned `api-key:Bearer [REDACTED]`. Surfaced by the post-merge security sitting, reproduced independently; it comes from the PR's rule order, not from `a3e164b5`, and is **not fixed**. A plain `Authorization: Bearer …` is unaffected. Fix idea: let the header value skip a following `Bearer`/`Basic` label, or run those rules first. *(low · packages/llm/src/scrub-secrets.ts)* -- [ ] **Scheduled → Wave 3 PR95. URL userinfo containing a raw `?` or `#` before the first `:` (or with no `:`) is not masked.** *Ruled 2026-10-19:* mask unless the text before the `?`/`#` is host-shaped (a dotted hostname, `localhost` or a bracketed IP, with an optional `:port`) **and** an `=` assignment lies between the `?`/`#` and the authority `@`; a plain host-shaped exemption would still leave `https://my.user:1234?TOKEN@h/` open. The same predicate applies to core's twin in `packages/core/src/tools/bounding.ts` (its colonless short-handle rule and `]` delimiter stay, recorded as differences). *Low (security).* *(author, from two independent differential runs)* `https://us?er:TOKEN@h/`, `http://#TOKEN@h` and `https://my.user:1234?TOKEN@h/` pass through unchanged; the pre-PR matcher masked all of them. It follows from the deliberate host-query policy in `maskUrlUserinfo` (a raw `?`/`#` may begin public query text, e.g. `https://host?contact=a@b.c`) and the colon-first cases are pinned by tests; it is identical at `19c99968` and is not introduced by the follow-up. **A maintainer call** whether a username shaped that way is worth the false positives. Fix idea: exempt only when the text before the `?`/`#` is non-empty and host-shaped (contains `.`, a `:port`, `localhost` or a bracketed IPv6), otherwise mask. *(low · packages/llm/src/scrub-secrets.ts)* -- [ ] **Scheduled → Wave 3 PR95. Credentials in URLs of other schemes are outside the finite rule.** *Ruled 2026-10-19:* extend the finite list with postgres, postgresql, mysql, mariadb, redis, rediss, mongodb, mongodb+srv, amqp, amqps, ssh, sftp, ldap, ldaps, smtp and smtps, masking only colon-bearing userinfo for the new schemes; malformed `https:/u:p@h`, a raw `/` inside a password and invisible characters inside the scheme stay documented limits. *Low, informational.* `postgres://`, `redis://`, `mongodb(+srv)://`, `ssh://` userinfo, malformed `https:/u:p@h`, a raw `/` inside a password, and invisible characters inside the scheme are not masked, identically before and after the follow-up. The file states its shapes are finite; a database connection error is the realistic carrier, so the scheme list may deserve a decision. *(info · packages/llm/src/scrub-secrets.ts)* -- [ ] **Scheduled → Wave 3 PR95. The URL rule's split-credential scan costs 2.1–2.5× on letter-heavy input, and nothing times that loop.** *Low / nit.* With `NO_BOUNDARY` the scanner tests every ASCII letter start for the URL rule; at 8 MiB the worst case stays under about 1.1 s and linear (1–16 MiB scaling checked). A prefilter on `h`/`w`/`f` would recover most of it. The existing 8 MiB timing case contains no `@`, so it returns before the loop; a guard would use a leading `a\nhttps://u@h ` plus 8 MiB of letters, but see the load-sensitive timing item below before adding another wall-clock assertion. SonarCloud was not run on the never-matching `NO_BOUNDARY = /(?!)/`; check its next analysis. *(low · packages/llm/src/scrub-secrets.ts)* -- [ ] **Scheduled → Wave 3 PR95. Two latent invariants in `hasSplitCredential` are unchecked.** *Nit.* A boundary-bearing rule with neither `\b` nor a `joinedPattern` makes `scrubSecrets` throw a `SyntaxError` on LF/TAB input no earlier rule has redacted (the suite fails at once, so it is caught, but it would break `makeLlmError`'s no-throw claim); and `wordBlocksStart` tests `boundary.test('a')` as a proxy for "rejects every ASCII word character", true for all current boundaries. An assertion at module load would pin both. *(nit · packages/llm/src/scrub-secrets.ts)* -- [ ] **Scheduled → Wave 3 PR95. The userinfo control's joined-view half is not separately witnessed.** *Low (coverage).* *(author, from an independent run)* Mutating only `joinedPattern` of the URL rule leaves 111/111 tests green. The whitespace-stripped residual probe independently catches the class the existing cases use, so it may be largely an equivalent mutant; an added case that did not discriminate it was reverted rather than kept. No causal control pins the glued-scheme behaviour either — only ordinary tests do. *(low · tools/secrets-controls/mutations.mjs)* +- **DONE 2026-10-22 — Wave 3 PR95 increment, closed under the maintainer's round cap. An `api-key` header label erases the `Bearer`/`Basic` label that follows it, so the token leaks.** *Low (security).* *(author)* `scrubSecrets('api-key:Bearer abcdefghijkl123456')` returns `api-key:[REDACTED] abcdefghijkl123456` — the `api-key` rule consumes the word `Bearer` as the header's value and the Bearer rule then has no label to anchor on. The pre-PR matcher returned `api-key:Bearer [REDACTED]`. Surfaced by the post-merge security sitting, reproduced independently; it comes from the PR's rule order, not from `a3e164b5`, and is **not fixed**. A plain `Authorization: Bearer …` is unaffected. Fix idea: let the header value skip a following `Bearer`/`Basic` label, or run those rules first. *(low · packages/llm/src/scrub-secrets.ts)* +- **DONE 2026-10-22 — Wave 3 PR95 increment, closed under the maintainer's round cap. URL userinfo containing a raw `?` or `#` before the first `:` (or with no `:`) is not masked.** *Ruled 2026-10-19:* mask unless the text before the `?`/`#` is host-shaped (a dotted hostname, `localhost` or a bracketed IP, with an optional `:port`) **and** an `=` assignment lies between the `?`/`#` and the authority `@`; a plain host-shaped exemption would still leave `https://my.user:1234?TOKEN@h/` open. The same predicate applies to core's twin in `packages/core/src/tools/bounding.ts` (its colonless short-handle rule and `]` delimiter stay, recorded as differences). *Low (security).* *(author, from two independent differential runs)* `https://us?er:TOKEN@h/`, `http://#TOKEN@h` and `https://my.user:1234?TOKEN@h/` pass through unchanged; the pre-PR matcher masked all of them. It follows from the deliberate host-query policy in `maskUrlUserinfo` (a raw `?`/`#` may begin public query text, e.g. `https://host?contact=a@b.c`) and the colon-first cases are pinned by tests; it is identical at `19c99968` and is not introduced by the follow-up. **A maintainer call** whether a username shaped that way is worth the false positives. Fix idea: exempt only when the text before the `?`/`#` is non-empty and host-shaped (contains `.`, a `:port`, `localhost` or a bracketed IPv6), otherwise mask. *(low · packages/llm/src/scrub-secrets.ts)* +- **DONE 2026-10-22 — Wave 3 PR95 increment, closed under the maintainer's round cap. Credentials in URLs of other schemes are outside the finite rule.** *Ruled 2026-10-19:* extend the finite list with postgres, postgresql, mysql, mariadb, redis, rediss, mongodb, mongodb+srv, amqp, amqps, ssh, sftp, ldap, ldaps, smtp and smtps, masking only colon-bearing userinfo for the new schemes; malformed `https:/u:p@h`, a raw `/` inside a password and invisible characters inside the scheme stay documented limits. *Low, informational.* `postgres://`, `redis://`, `mongodb(+srv)://`, `ssh://` userinfo, malformed `https:/u:p@h`, a raw `/` inside a password, and invisible characters inside the scheme are not masked, identically before and after the follow-up. The file states its shapes are finite; a database connection error is the realistic carrier, so the scheme list may deserve a decision. *(info · packages/llm/src/scrub-secrets.ts)* +- **DONE 2026-10-22 — Wave 3 PR95 increment, closed under the maintainer's round cap. The URL rule's split-credential scan costs 2.1–2.5× on letter-heavy input, and nothing times that loop.** *Low / nit.* With `NO_BOUNDARY` the scanner tests every ASCII letter start for the URL rule; at 8 MiB the worst case stays under about 1.1 s and linear (1–16 MiB scaling checked). A prefilter on `h`/`w`/`f` would recover most of it. The existing 8 MiB timing case contains no `@`, so it returns before the loop; a guard would use a leading `a\nhttps://u@h ` plus 8 MiB of letters, but see the load-sensitive timing item below before adding another wall-clock assertion. SonarCloud was not run on the never-matching `NO_BOUNDARY = /(?!)/`; check its next analysis. *(low · packages/llm/src/scrub-secrets.ts)* +- **DONE 2026-10-22 — Wave 3 PR95 increment, closed under the maintainer's round cap. Two latent invariants in `hasSplitCredential` are unchecked.** *Nit.* A boundary-bearing rule with neither `\b` nor a `joinedPattern` makes `scrubSecrets` throw a `SyntaxError` on LF/TAB input no earlier rule has redacted (the suite fails at once, so it is caught, but it would break `makeLlmError`'s no-throw claim); and `wordBlocksStart` tests `boundary.test('a')` as a proxy for "rejects every ASCII word character", true for all current boundaries. An assertion at module load would pin both. *(nit · packages/llm/src/scrub-secrets.ts)* +- **DONE 2026-10-22 — Wave 3 PR95 increment, closed under the maintainer's round cap. The userinfo control's joined-view half is not separately witnessed.** *Low (coverage).* *(author, from an independent run)* Mutating only `joinedPattern` of the URL rule leaves 111/111 tests green. The whitespace-stripped residual probe independently catches the class the existing cases use, so it may be largely an equivalent mutant; an added case that did not discriminate it was reverted rather than kept. No causal control pins the glued-scheme behaviour either — only ordinary tests do. *(low · tools/secrets-controls/mutations.mjs)* - [ ] **Approved deferral (maintainer, 2026-10-19). `LOW-2` (the 1 s default `waitFor` on a real child) was examined and deliberately not changed.** *Low.* The only wait that observes a real child booting is `holdNativeToolProcess`'s marker wait, whose sole caller sits inside `drive-home.test.ts` waits (lines 633 and 637) that use the 1 s default and enclose the boot, so a longer inner bound cannot help; a retune must raise those outer waits together. No failure was reproduced (0 in dozens of loaded runs). `testing.md` also states that no readiness or case deadline is extended to pass the workspace gate. **Trigger: a recorded boot-time failure.** *(low · apps/cli/src/commands/native-forwarding.test-support.ts, apps/cli/src/home/drive-home.test.ts)* - [ ] **Scheduled → Wave 3 17b. Join-table test nits.** *Nit.* The `process-close.test.ts` gate margin is nominally 100 ms but `waitFor`'s 50 ms poll makes it 100–150 ms, so a publication more than about 150 ms late is invisible; the three rejection cases assert the message but not the `ProcessCapabilityError` class; the `mcp-orphan.e2e.test.ts` comment's statement that its cases have only ever run on POSIX is historical and unverifiable; the abort case's `await spawned.promise` is unbounded (a spawn that never happens fails only as the 5 s case timeout), its closed-child `waitFor` carries no tag, and the 100/300/3,000 ms margins are bare literals. *(nit · apps/cli/src/engine/tool-host/process-close.test.ts, apps/cli/src/harness/mcp-orphan.e2e.test.ts)* - [ ] **Approved deferral (maintainer, 2026-10-19). Native-lifetime watchdogs are fixed 5 s bounds inside 10 s cases (`LOW-1`).** *Low.* The watchdogs in @@ -410,7 +410,7 @@ measured or counted here. does nothing, and three runs of the three vitest files (52 tests) passed at a load average of 80–96 on macOS. The reviewer could not observe the Windows root cause either. **Trigger: a Windows trace that names the bound that fired**; retune the coupled set together then. *(low · the files above)* -- [ ] **Scheduled → Wave 3 PR95. `bounds admitted-size failed header and URL preparation` is load-sensitive.** *Low.* *(author)* It allows +- **DONE 2026-10-22 — Wave 3 PR95 increment, closed under the maintainer's round cap. `bounds admitted-size failed header and URL preparation` is load-sensitive.** *Low.* *(author)* It allows 5 s for two 8 MiB scrubs that take about 0.75 s in total, and it failed twice with the load average near 65 (6.4 s in the run that logged the duration). Measured alternately, the pre-change code and the corrected code take the same time, so the failures were load, not the correction. Two independent reviewers saw it too (5,590 ms and 3,453 ms at load averages near 88–97) and recorded that `tools/secrets-controls/README.md` says the fixture runs without a machine-speed threshold, which this assertion contradicts; six controls run this file, so a loaded runner can turn a fixed phase red. The test @@ -448,14 +448,17 @@ measured or counted here. can return a raw signal code above 9 that the physical exit never uses, a note for future embedders. No action is proposed for any of these. -## PR #95 masker low edge cases — approved deferrals (2026-10-21) +## PR #95 masker edge cases — approved deferrals (2026-10-21 and 2026-10-22) Under the [masker acceptance bar](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20) that the maintainer set on 2026-10-21, a low finding in the open-ended masker work (`packages/llm/src/scrub-secrets.ts` and core's twin projection in `packages/core/src/tools/bounding.ts`) is -recorded here instead of being corrected in the review loop. None of these entries is a regression against base -`9902e0a0`, and none was judged high or medium; the two entries marked pending were found by the author after -round 9, outside any finding, and await the maintainer's confirmation after round 10. The shapes that neither masker reads by design stay described in +recorded here instead of being corrected in the review loop, and under the round cap of the same day so is +every medium and low finding of round 10, the last round, after which only high findings are corrected. The +entries up to round 9 are not regressions against base `9902e0a0`; the round-10 entries say when they are, in the +shapes they name. The maintainer counted round 10's GEN-R10-1 (a `.env` password in a structured result's +summary) as high on 2026-10-22, so its summary half is corrected and only its journal half is recorded here, and +approved the two private-key block gaps the author found after round 9. The shapes that neither masker reads by design stay described in their canonical homes, [built-in-tools.md](../reference/shared-core/built-in-tools.md) for core and [error-handling.md](../standards/error-handling.md) for the scrubber; this list holds the review findings and the re-checks they need. **Trigger for every entry:** a real tool result, journal entry or diagnostic that carries the @@ -497,23 +500,73 @@ the repository, so each entry names the input and the property to re-check by ha `tools/secrets-controls/mutations.mjs` targets it. **Re-check:** add a control (for example `keepingEscapes` returning `ranges`) when the secrets family is next extended, with its counts. *(low · tools/secrets-controls/mutations.mjs)* -- [ ] **Deferral pending the maintainer's confirmation (author, 2026-10-21). A private-key block cut off before - its END marker is not masked.** *Unchanged from base; found by the author after round 9, outside any review - finding.* `redactSecretShapedText('\nMIIE…')` and the summary of the same text keep the whole - body, as does a block whose END marker starts more than 20,000 characters after its BEGIN marker - (`PEM_BODY_MAX`): `visitPemBlocks` masks only a BEGIN…END pair, and the LLM backstop reads no private-key - block. A real tool output can cut a key short (`head -n 5 key.pem`, a tool that clips its own output). - built-in-tools.md names the limit. **Re-check:** mask an unclosed BEGIN marker through its base64 body lines in - every projection, linear at admitted size; a journal change needs the compatibility review of +- [ ] **Approved deferral (maintainer, 2026-10-22). A private-key block cut off before its END marker is not + masked.** *Unchanged from base; found by the author after round 9, outside any review finding.* + `redactSecretShapedText('\nMIIE…')` and the summary of the same text keep the whole body, as does + a block whose END marker starts more than 20,000 characters after its BEGIN marker (`PEM_BODY_MAX`): + `visitPemBlocks` masks only a BEGIN…END pair, and the LLM backstop reads no private-key block. A real tool output + can cut a key short (`head -n 5 key.pem`, a tool that clips its own output). built-in-tools.md names the limit. + **Re-check:** mask an unclosed BEGIN marker through its base64 body lines in every projection, linear at admitted + size; a journal change needs the compatibility review of + [effect-journal.md](../reference/shared-core/effect-journal.md#11-secrets-what-a-row-may-hold) §11. + *(packages/core/src/tools/bounding.ts)* +- [ ] **Approved deferral (maintainer, 2026-10-22). A structured result's summary does not read a private-key + marker split by a line break or tab.** *Unchanged from base; found by the author after round 9.* + `boundForModel({ a: '\nMIIE…\n-----END RSA PRIVATE\nKEY-----' })` shows the block in its summary, + while the same text as a string result is `[redacted]`: JSON escapes the line break or tab, so the wrapped-marker + patterns, which read raw whitespace, see none; more than one space still collapses in the summary's one-line + pass. built-in-tools.md names the limit. **Re-check:** read the wrapped markers in `escapedSeparatorView` as well + (`visitEscapedSeparators`). *(packages/core/src/tools/bounding.ts)* +- [ ] **Approved deferral (round cap, 2026-10-21). A summary reads a wrapped private-key marker only within its + whitespace bounds.** *Medium, regression against base.* *(round 10, R10-SCRUB-1)* + `boundForModel('PRIVATE_KEY=-----BEGIN PRIVATE KEY-----\nMIIE…\n-----END PRIVATE\n KEY-----')` shows the + body in its summary, which base masked: the wrapped patterns take one to eight whitespace characters between + `PRIVATE` and `KEY` and up to 40 characters of type words, and the key's unquoted value takes the BEGIN marker + before the one-line pass can pair the block. A line break with seven or more spaces of indentation in CRLF text, + nine tabs and a 31-column hanging indent leak the same way. built-in-tools.md names the bound. **Re-check:** let + each whitespace run grow as one simple quantifier (`PRIVATE\s+KEY`, type words separated by `\s+` with a bounded + word count) under the admitted-size case deadline, and add summary tests with nine or more whitespace characters + behind a key prefix. *(medium · packages/core/src/tools/bounding.ts `visitPemBlocks`)* +- [ ] **Approved deferral (round cap, 2026-10-21). A U+E000 in a structured result brings back walk-first masking + in its summary.** *Medium, regression against base.* *(round 10, R10-SCRUB-2, INT-R10-1 and PR95-R10-SEC-1)* + `boundForModel({ icon: ' ok', private_key: '\nMIIEAAAAfirst', rest: 'MIIEBBBBsecond\n' })` + shows the second half of the body and the END marker in its summary; `{ argv: ['mysql', "--password='correct", + 'horse', "battery'"], icon: '' }` shows `horse` and `battery'`. The summary marks each whole-masked value with + a U+E000 placeholder and, when the result's JSON already holds that character, falls back to the walk's masks + applied before the text rules. A planted character or a glyph (a Nerd Font icon) is enough. built-in-tools.md + names the limit. **Re-check:** pick a placeholder that the result's JSON does not hold (an unused private-use code + point or a nonce) so that the unwalked reading always runs, and add tests with U+E000 beside a cross-member PEM + block and an argv password. *(medium · packages/core/src/tools/bounding.ts `structuredSummaryText`)* +- [ ] **Approved deferral (round cap, 2026-10-21). A private-key block ends at its first END marker even where + another rule's match is glued to it.** *Medium, regression against base since `b89ac62e`.* *(round 10, + R10-SCRUB-3)* `redactSecretShapedText("api_key='x Wsecret12 -----BEGIN RSA PRIVATE KEY-----' ?sig=<24 a>-----END RSA PRIVATE KEY-----")` + is `api_key='x Wsecret12 [redacted]` (base: `[redacted] ?sig=[redacted] RSA PRIVATE KEY-----`), and + `'\n?s=<26 characters>W'` leaves `W`, which base masked; base's + rules ran in turn, so the glued match erased the marker. built-in-tools.md names the relaxation. **Re-check:** end + a block only at an END marker that no other rule's match is glued to, or read a value's closing quote inside a + PEM range as closing. *(medium · packages/core/src/tools/bounding.ts `visitPemBlocks`, `visitSecretKeyValues`)* +- [ ] **Approved deferral (round cap, 2026-10-21). Whitespace inside a wrapped marker's type words has no killing + test.** *Medium, test gap.* *(round 10, TST-R10-1)* Changing either wrapped pattern's type class + `[A-Z0-9\s]{0,40}` to `[A-Z0-9 ]{0,40}` leaves every bounding case passing, while the summary of + `PRIVATE_KEY=-----BEGIN RSA\nPRIVATE KEY-----\nMIIE…\n-----END RSA PRIVATE KEY-----`, or of the same split in the + END marker, then shows the body. **Re-check:** add both inputs to `reads a private-key marker that a line break, a + tab or a doubled space splits in a summary` and run both class mutants. + *(medium · packages/core/src/tools/bounding.test.ts)* +- [ ] **Approved deferral (round cap, 2026-10-21). The summary's no-throw handling of a whole-masked value with no + JSON form has no killing test.** *Low, test gap.* *(round 10, TST-R10-2)* With `jsonValue` reduced to + `return JSON.stringify(value);`, `boundForModel({ password: 10n, ok: 1 })` and + `boundForModel(['--password', 10n, 'x'])` reject and every test still passes; the existing no-JSON-form test holds + no credential key. **Re-check:** add both inputs (summaries `{"[redacted],"ok":1}` and + `["--password","[redacted]","x"]`) and run the mutant. *(low · packages/core/src/tools/bounding.test.ts)* +- [ ] **Approved deferral (maintainer, 2026-10-22). The journal projection of JSON text keeps a value whose escaped + opening quote an earlier mask took.** *Unchanged from base; the journal half of GEN-R10-1, whose summary half is + corrected.* `redactSecretShapedText(JSON.stringify({ stdout: 'DATABASE_URL=postgres://db.example.com/app?sslmode=require\nPOSTGRES_PASSWORD=""\n' }))` + shows ``: the query value runs on through the JSON-escaped line break and the next line's key and takes the + backslash of the value's escaped quote. A summary opens such a value where that mask keeps its backslashes; the + journal does not, so `v5:` digests do not change, and a test pins that. **Re-check:** drop the `throughCloser` + gate of the kept opening in `visitSecretKeyValues`, with the compatibility review of [effect-journal.md](../reference/shared-core/effect-journal.md#11-secrets-what-a-row-may-hold) §11. *(packages/core/src/tools/bounding.ts)* -- [ ] **Deferral pending the maintainer's confirmation (author, 2026-10-21). A structured result's summary does - not read a private-key marker split by a line break or tab.** *Unchanged from base; found by the author after - round 9.* `boundForModel({ a: '\nMIIE…\n-----END RSA PRIVATE\nKEY-----' })` shows the block in - its summary, while the same text as a string result is `[redacted]`: JSON escapes the line break or tab, so the - wrapped-marker patterns, which read raw whitespace, see none; more than one space still collapses in the - summary's one-line pass. built-in-tools.md names the limit. **Re-check:** read the wrapped markers in - `escapedSeparatorView` as well (`visitEscapedSeparators`). *(packages/core/src/tools/bounding.ts)* ## Phase 2.6.5 deferrals — `CR-93` and `CR-95` (long half), recorded 2026-09-06; `CR-70`, `CR-94` (remainder) and the in-turn overflow continuation, recorded 2026-09-14; the errored-or-aborted turn residual, the pre-upgrade free-page bytes and the estimator's calibration, recorded 2026-09-18 diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 241cc71c..d5146fa6 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -3694,7 +3694,7 @@ No credential values or request captures belong in tracked evidence. | 8b | **Recorded split — creation-owned native processes:** tool POSIX group versus MCP PID, actual close/retirement, verified absolute Windows teardown admission and helper lifetime; connect OS/editor helpers and replace independent reapers without claiming full descendant containment. | | 8c | **Recorded split — workflow and render integration:** run/gate/budget/driver owners, real return/throw health priority, cancellation, terminal/job-control leases and force-quit routing. No timer certifies DB close or receipt settlement. | | 8d | **Recorded split — sessions, terminal and activation:** fresh/resumed/one-shot/Home/pending/replaced session lifetimes, retained publication failure and shared actual cleanup promises; connect caught slash/rebuild faults, restore terminal custody before diagnosis, then activate early coordinator/background/sink integration. Actual native/platform/packed controls and complete cumulative Step 8 acceptance follow. | -| PR95 | **Recorded plan amendment, 2026-10-19 (maintainer-approved; new scope from the [PR #95 independent review](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18), not a split of rows 1–19) — scrubber residuals, before 9a:** the `api-key:Bearer`/`Basic` label erasure that leaks the token; raw `?`/`#` URL userinfo masked unless the text before it is host-shaped **and** an `=` assignment separates it from the authority `@`, applied to the LLM leaf and core's twin predicate; an extended finite scheme list (postgres, postgresql, mysql, mariadb, redis, rediss, mongodb, mongodb+srv, amqp, amqps, ssh, sftp, ldap, ldaps, smtp, smtps) masking colon-bearing userinfo for the new schemes; module-load checks of the split-credential scanner's two invariants, a start prefilter, joined-view and glued-scheme witnesses, the load-sensitive 8 MiB timing case, and the causal-control pins. One increment with ordered commits, two NEW clean complete cumulative rounds and a separate security sitting. *Acceptance amended on 2026-10-21 by the maintainer's masker acceptance bar and round cap ([checkpoint](#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)).* | +| PR95 | **Recorded plan amendment, 2026-10-19 (maintainer-approved; new scope from the [PR #95 independent review](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18), not a split of rows 1–19) — scrubber residuals, before 9a:** the `api-key:Bearer`/`Basic` label erasure that leaks the token; raw `?`/`#` URL userinfo masked unless the text before it is host-shaped **and** an `=` assignment separates it from the authority `@`, applied to the LLM leaf and core's twin predicate; an extended finite scheme list (postgres, postgresql, mysql, mariadb, redis, rediss, mongodb, mongodb+srv, amqp, amqps, ssh, sftp, ldap, ldaps, smtp, smtps) masking colon-bearing userinfo for the new schemes; module-load checks of the split-credential scanner's two invariants, a start prefilter, joined-view and glued-scheme witnesses, the load-sensitive 8 MiB timing case, and the causal-control pins. One increment with ordered commits, two NEW clean complete cumulative rounds and a separate security sitting. *Acceptance amended on 2026-10-21 by the maintainer's masker acceptance bar and round cap; closed under the round cap on 2026-10-22 ([checkpoint](#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)).* | | 9 | **Release map/custody substrate:** #46/ADR-0111, independently reviewed after the finite graph. Upstream TS maps, sole prepack final build, pre-hash trailer removal/staging, exact tarball/CLI+engine symbolication, encrypt-once candidate with permissioned prebuild discovery/reservation → pack/reuse → smoke → archive → npm → finalization. Required artifact/platform aggregation, all rerun modes, lost-staging refusal and permission-enforcing offline API controls. Dedicated archive/crypto security sitting. Fixture proof never claims production recipient/custody configuration. | | 9a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage #46/ADR-0111 unchanged, no new contract) — release maps, final build and symbolication:** fresh uncached upstream original-TypeScript maps, the sole prepack final CLI build, final trailer removal before hashes, map and private-graph staging outside public artifacts and caches, exact tarball-to-final-JS binding, and real packed CLI and engine frame symbolication with wrong JS/map/source and missing-map negative controls. Ordinary builds remain map-free. *Split again on 2026-10-19 into 9a.1 and 9a.2 below, which replace it as increments ([Plan shape](#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19)).* | | 9a.1 | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — candidate build and finalizer:** an owned isolated candidate copy, fresh uncached upstream maps, candidate-mode tsup selection with absent `sourcesContent`, the prepack finalizer (upstream and final map validation, repository-relative sources, exact trailer removal before hashes), owner-only private staging outside public artifacts and caches, closure/seal adaptation, the ordinary map-free control, the PR #95 archive/worker fold-ins, and a fail-closed `release.yml` publish guard that refuses any non-release candidate until 9d and Wave 4b custody exist. The receipt stays `compiled_package_test`. | @@ -9424,8 +9424,8 @@ carries the same owners. the CLI's authored-content detector, the secrets causal controls and the canonical documents. Base `9902e0a0`; commits `38e10d42` (Docs-0, the rulings), `dda3b45e` (implementation) and the corrections `ef3ded0d`, `bdd292a6`, `6fb7e9d9`, `b2c50e88`, `6b2e94dd` (round 5, which added this -section), `b89ac62e` (round 6), `082962e2` (round 7) and the round-8 commit. The commits stay local on -`development` until the increment is accepted. +section), `b89ac62e` (round 6), `082962e2` (round 7), `6f7dd342` (round 8), `59c64498` (round 9) and the +round-10 commit. The commits stayed local on `development` until the increment closed. **Review rounds.** Each round is a NEW independent complete cumulative review of `9902e0a0..HEAD`, every reviewer in its own worktree detached at the reviewed head: scrubber correctness, tests and causal @@ -9444,6 +9444,7 @@ stay local and Git-ignored. | 7 | `b89ac62e` | NONCLEAN | 17 (2 high) | only a userinfo's username owns its keys, never past an LF or TAB, so a key in its password or parameter position or on the next line masks its value through the last `@`; quoted values are found by a character scan that an admitted 8-MiB value cannot exhaust; a credential glued to a fixed-length one is masked in the same pass; summaries use the projection's fixed point; a tiered call whose journal projection does not settle is refused before its prepare; the secrets controls run as two CI shards so that each evidence export fits the unchanged 64-MiB bound, which the single export had exceeded; untested pieces gained discriminating tests | | 8 | `082962e2` | NONCLEAN (integration and security clean) | 16 (1 high) | round 7's glued-credential chain cut wider credentials and swallowed key names, and is gone; a structured result's summary reads its JSON as one text once credential members are masked; quoted values are read one JSON string down, and one that does not close on its line is read as an unquoted run from after its quote, no further than that line's end (one JSON string down, its string's end), unless another mask of the pass takes its line break and it closes; a summary masks one stopped by a line break through a later quote instead; a header label at a line start inside a split continuation is read; the scrubber's split probe also reads the raw text; a password-position key's extent is documented as credential equality. Before the commit the author ran the reviewers' method on the candidate and six internal pre-reviews: the first found three of the draft's alternatives unsound (a trusted host-shaped tail, quoted values across lines, a quadratic line-start scan), and they were dropped; the second (17 findings, none high) led to the summary's through-the-closer masking, colliding member names refused as unsettled and escaped separators and opaque values before escaped quotes; the third (27 findings, two high) found that the round's own additions failed — masking a value that never closes to its line end merged journal identities, and the scrubber's escaped header separator and terminal final-byte view let a token leak or blanked coloured diagnostics — so such a value is read as an unquoted run, the two scrubber additions are withdrawn and documented as limits, and a structured summary's walk became linear at a small constant factor; the fourth (16 findings, two medium; the critic clean, with all 128 controls run) found that the run and the opaque mask's kept backslash showed values that earlier versions masked whole in their next pass, so such a value is decided once the pass's other masks are known and the opaque mask takes the backslash again, and gaps in tests and wording; the fifth (19 findings, one high; the critic clean) found that projecting a structured result's strings one by one removed the PEM markers and keys that reading its JSON needs across members, so a structured summary reads its JSON as one text again and the summary-only runs over line breaks are withdrawn; the sixth (four findings and a duplicate, one medium; the integration lens and the critic clean) found that the closing view and the summary's closer read a quote's escape after a nested key's value had taken its backslash, and that a structured summary's walk dropped the private-key markers a block across members needs, both regressions against base, and that a structured summary read no JSON-escaped line break or tab, a gap base shares, so a quoted value closes at the later of its readings with and without the masks' trailing backslashes, which also masks the tail an opaque value's mask unescaped (behind a URL userinfo, a further regression the author's oracles found), the walk keeps a marker whose partner lies outside its member, and every summary also reads its escaped separators; two low edge cases base shares went to deferred-tasks.md under the maintainer's 2026-10-21 bar | | 9 | `6f7dd342` | NONCLEAN (integration clean) | 18 (2 high, 8 medium) | round 8's view comparisons and per-range backslash walks were quadratic at admitted size, so each view is built once and a shared backslash run is read once; a quoted value is also read with the scheme labels' tokens filled, and one that does not close on its line with and without the key rule's own masks; a summary masks a value that one reading stops at a line break through its later closer; a structured summary reads the unwalked JSON once with the walk's whole masks applied in that pass, which replaces round 8's kept PEM markers; a summary reads private-key markers that a line break, a tab or more than one space splits; three low findings went to deferred-tasks.md, and two private-key block gaps the author found afterwards, unchanged since base, are named as limits and recorded there pending the maintainer's confirmation | +| 10 | `59c64498` | NONCLEAN (critic clean) | 11 (7 medium and 4 low as verified; three of the mediums are one finding) | no high or blocker finding; the maintainer counted GEN-R10-1 as high on 2026-10-22 — a `cat .env` result whose password, after a query value at the end of a line, showed in its structured summary, unchanged since base — so a summary opens a key's value whose escaped opening quote ends an earlier mask's backslashes where that mask keeps them, and the journal keeps its reading, so `v5:` digests do not change; the docblocks and the canonical sentence that counted one summary difference, effect-journal.md §11's closing readings and the status records are corrected; the wrapped markers' whitespace bound, the U+E000 placeholder's fallback, a PEM END marker glued to another rule's match, two test gaps and the journal half of GEN-R10-1 are named as limits where a canonical document claimed otherwise and recorded in deferred-tasks.md under the round cap; the two private-key block gaps found after round 9 are approved deferrals | **Author gates and controls.** Every corrected head passed `pnpm run ci` and the affected suites under `CI=true GITHUB_ACTIONS=true`, plus local differential, exhaustive and admitted-size harnesses @@ -9457,7 +9458,7 @@ project 1,087 of the unchanged 2,000 retained files; both shards ran locally wit `RELAVIUM_ACCEPTANCE_EVIDENCE` set, and each export held 1,087 files and measured 45.50 MiB and 45.48 MiB, its final record included. From round 8 the family is 130 controls / 390 phases / 513 targets and witnesses in two shards of 65 controls (1,177 files each), whose exports measured -49.83 MiB and 49.76 MiB (50.19 MiB and 50.09 MiB at the round-9 source). From round 8 the author also runs the reviewers' own +49.83 MiB and 49.76 MiB (50.19 MiB and 50.09 MiB at the round-9 source, 50.24 MiB and 50.14 MiB at the round-10 source). From round 8 the author also runs the reviewers' own generative harness (a planted-token oracle, idempotence, JSON round trips, prose preservation, admitted-size timing and a search for projections that do not settle) and an internal multi-lens pre-review on every correction before its commit. @@ -9471,22 +9472,41 @@ cannot be complete, and each aggressive round found another corner case while St is clean for this increment when it reports no high or medium finding and no regression against base `9902e0a0` (text that base masked and the reviewed head shows, or a guarantee base kept, such as no throw, linear time at admitted size or a settled projection, that the reviewed head no longer keeps). Each low finding is recorded in -[deferred-tasks.md](../deferred-tasks.md#pr-95-masker-low-edge-cases--approved-deferrals-2026-10-21) +[deferred-tasks.md](../deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22) with its file, a minimal example and where to re-check it, instead of being corrected in the loop. -The increment closes with the sixth internal pre-review and one external round (round 9) under that -bar. The ruling covers only the open-ended masker work: `packages/llm/src/scrub-secrets.ts`, core's +The increment was to close with the sixth internal pre-review and one external round (round 9) under +that bar; the round cap below superseded that. The ruling covers only the open-ended masker work: `packages/llm/src/scrub-secrets.ts`, core's twin projection in `packages/core/src/tools/bounding.ts` and their controls. Every other increment keeps the approved bar of two NEW consecutive clean complete cumulative rounds after the last material fix. **Maintainer ruling — the round cap, 2026-10-21.** Round 9 reported two high findings, in round 8's own quadratic view comparisons, and eight medium regressions in five constructed shapes. The maintainer -ruled that every round-9 finding is corrected and round 10 runs, and that the loop then stops: after -round 10, whatever it reports, only its high findings are corrected; its medium and low findings are -recorded in deferred-tasks.md with the file, a minimal example and where to re-check it, and the plan -moves on to Step 9a.1. - -**Remaining gates.** The increment is not independently accepted. Round 10 remains: a NEW complete -cumulative round with its separate security lens, under the round cap. Then `development` is pushed -and its CI observed, and the PR #95 items in -[deferred-tasks.md](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18) close. +ruled that round 9's high and medium findings are corrected (its three low findings were recorded under +the bar) and round 10 runs, and that the loop then stops: after round 10, whatever it reports, only its +high and blocker findings are corrected; its medium and low findings are recorded in deferred-tasks.md +with the file, a minimal example and where to re-check it, and the plan moves on to Step 9a.1. + +**Maintainer rulings after round 10, 2026-10-22.** Round 10 reported no high or blocker finding. The +maintainer counted GEN-R10-1 as high, since a `cat .env` run_command or MCP result whose +`POSTGRES_PASSWORD` line follows a URL ending in a query value showed that password in the structured +summary that the `agent:tool_result` event carries into history.db (unchanged since base), and ruled +that it is corrected in the summary only, so that `v5:` journal digests do not change. The maintainer +also approved the two private-key block gaps the author found after round 9 as deferrals. As the round +cap provides, the correction has author verification only: a test with five witnesses and a journal pin +(218 bounding cases), a mutation run of six mutants on the new code that kills four (the two survivors +are output-neutral, because a mask that the key rule's kept view keeps ends before a delimiter, where a +view that keeps the backslashes closes no later than the filled one), the eight controls next to the +changed code (`core-quoted-value-scan` and `core-escaped-quote` re-anchored), the full `pnpm run ci` +with both exporting shards of all 130 controls, and differentials against +`59c64498` and base: the journal and display text identical on 250,000 inputs, every summary that +differs on 320,000 inputs showing a subsequence of `59c64498`'s visible text, and no planted-token +regression against base or `59c64498` in the generators or in an exhaustive run of 837,606 sequences. + +**Closure, 2026-10-22.** The increment closes under the round cap with the round-10 correction. It is +not independently accepted in the sense of the approved bar: round 10 was NONCLEAN, and its correction +was verified by the author alone, as the round cap provides; every finding it did not correct is in +[deferred-tasks.md](../deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). +`development` is pushed and its CI observed, and the PR95 items in +[deferred-tasks.md](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18) are +closed. Step 9a.1 follows. diff --git a/packages/core/src/tools/bounding.test.ts b/packages/core/src/tools/bounding.test.ts index 56ca0073..2bd6194b 100644 --- a/packages/core/src/tools/bounding.test.ts +++ b/packages/core/src/tools/bounding.test.ts @@ -2999,3 +2999,55 @@ describe('PR95 review round 9: closing views, unwalked members, wrapped markers ADMITTED_SIZE_CASE_TIMEOUT_MS, ); }); + +describe('PR95 review round 10: a value whose escaped quote an earlier mask took, in a summary', () => { + it("opens a value whose escaped quote ends an earlier mask's backslashes in a summary", async () => { + // A structured result's JSON escapes each line break, so a query value at the end of a line runs on + // through the next line's key and takes the backslash of its value's escaped quote; the summary opens + // that value where the query value's mask keeps its trailing backslashes. + const env = + 'DATABASE_URL=postgres://db.example.com/app?sslmode=require\nPOSTGRES_PASSWORD="S3cr3tValue2024"\n'; + for (const [result, summary, secret] of [ + [ + { exitCode: 0, stdout: env, stderr: '', durationMs: 12 }, + '{"exitCode":0,"stdout":"DATABASE_URL=postgres://db.example.com/app?sslmode=[redacted]\\n","stderr":"","durationMs":12}', + 'S3cr3tValue2024', + ], + [ + { content: [{ type: 'text', text: env }] }, + '{"content":[{"type":"text","text":"DATABASE_URL=postgres://db.example.com/app?sslmode=[redacted]\\n"}]}', + 'S3cr3tValue2024', + ], + [ + { stdout: env.replaceAll('\n', '\r\n') }, + '{"stdout":"DATABASE_URL=postgres://db.example.com/app?sslmode=[redacted]\\r\\n"}', + 'S3cr3tValue2024', + ], + [ + { + stdout: + 'MONGO_URL=mongodb://h/db?retryWrites=true&w=majority\nMONGO_PASSWORD="M0ngoSecretPass"\n', + }, + '{"stdout":"MONGO_URL=mongodb://h/db?retryWrites=true&w=[redacted]\\n"}', + 'M0ngoSecretPass', + ], + [ + { stdout: 'GET /x?ssl_cert_reqs=required\tREDIS_PASSWORD="R3disSecretPass"' }, + '{"stdout":"GET /x?ssl_cert_reqs=[redacted]"}', + 'R3disSecretPass', + ], + ] as const) { + const bounded = await boundForModel(result, BIG, {}); + expect(bounded.summary, 'SECRETS_B10_CORE_SUMMARY_KEPT_OPENING').toBe(summary); + expect(bounded.summary.includes(secret), 'SECRETS_B10_CORE_SUMMARY_KEPT_OPENING').toBe(false); + } + // The journal projection, which is compared rather than shown, keeps its reading so that its digests do + // not change: the same JSON as a text argument still shows the value, a residual deferred-tasks.md records. + expect( + redactSecretShapedText(JSON.stringify({ stdout: env })), + 'SECRETS_B10_CORE_SUMMARY_KEPT_OPENING', + ).toBe( + '{"stdout":"DATABASE_URL=postgres://db.example.com/app?sslmode=[redacted]"S3cr3tValue2024\\"\\n"}', + ); + }); +}); diff --git a/packages/core/src/tools/bounding.ts b/packages/core/src/tools/bounding.ts index fe8f156d..4adf83a0 100644 --- a/packages/core/src/tools/bounding.ts +++ b/packages/core/src/tools/bounding.ts @@ -502,11 +502,14 @@ export function redactSecretShapedText(text: string): string { } /** - * The summary's projection (`makeSummary`). A summary is shown, never compared, so it differs in one decision - * (`visitSecretKeyValues`): a quoted value that stops at a line break is masked through a later quote that - * could close it, else to the end of its line, and one that reaches the end of its string or of the text - * without closing is masked to that end. Left to the one-line pass, such a value lost its key to a mask - * before it (`token='a password="first\nsecond" done`), and JSON escaping hid it in a structured result. + * The summary's projection (`makeSummary`). A summary is shown, never compared, so it decides three things + * differently. In the key rule (`visitSecretKeyValues`), a quoted value that stops at a line break is masked + * through a later quote that could close it, else to the end of its line, and one that reaches the end of + * its string or of the text without closing is masked to that end: left to the one-line pass, such a value + * lost its key to a mask before it (`token='a password="first\nsecond" done`), and JSON escaping hid it in a + * structured result; and a value whose escaped opening quote ends an earlier mask's backslashes opens where + * that mask keeps them. It also reads private-key markers that whitespace wraps (`visitPemBlocks`) and the + * rules a separator decides behind escaped separators (`visitEscapedSeparators`). */ function redactSecretShapedSummaryText(text: string): string { return toFixedPoint((value) => redactSecretShapedOnce(value, true), text, '[redacted]'); @@ -852,8 +855,10 @@ function valueDelimiter(): RegExp { * string around it, is returned for the journal projection to decide once the pass's other ranges are * known (`visitUnclosedValues`). The summary's projection (`throughCloser`) masks a value that one of its * readings stops at a line break through a later quote instead (`throughNearestCloser`), and through - * the closing of a reading that closes too. Keys are read in `keys`, which hides each URL userinfo's - * username (`UrlUserinfo.ownsUntil`): a key there names the user + * the closing of a reading that closes too; it also opens a value whose escaped opening quote ends an + * earlier mask's backslashes in the views that keep them (`?sslmode=require\nPOSTGRES_PASSWORD=\"…\"` in a + * structured result, where the query value runs on through the next line's key). Keys are read in + * `keys`, which hides each URL userinfo's username (`UrlUserinfo.ownsUntil`): a key there names the user * (`https://x-access-token:@github.com/o/r`), and its value would run on through the public host and * path. Values are read in `values`, where every earlier mask is one run, as an applied mask would be, * and a quoted value also in the other `views` (`quoteViews`); it closes at the latest of them @@ -884,11 +889,19 @@ function visitSecretKeyValues( }; for (let match = key.exec(keys); match !== null; match = key.exec(keys)) { const valueStart = escapedValueStart(values, key.lastIndex); - const escaped = values[valueStart] === '\\'; - const opening = escaped ? values[valueStart + 1] : values[valueStart]; + // In a summary, a value whose escaped opening quote ends an earlier mask's backslashes (a query value + // that ran on through a JSON-escaped line break and the next line's key) opens where that mask keeps + // them, and is read only in the views that keep them. + const kept = + throughCloser && values[valueStart] !== '\\' && views.escapes?.[valueStart] === '\\' + ? views.escapes + : undefined; + const opener = kept ?? values; + const escaped = opener[valueStart] === '\\'; + const opening = escaped ? opener[valueStart + 1] : opener[valueStart]; if (opening === '"' || opening === "'") { - const value = quotedValueEnd(values, valueStart); - const readings = [value, ...views.plain.map((view) => quotedValueEnd(view, valueStart))]; + const plain = views.plain.map((view) => quotedValueEnd(view, valueStart)); + const readings = kept === undefined ? [quotedValueEnd(values, valueStart), ...plain] : plain; // A scheme label's token can take the line break before a closer that a reading reaches early; a // value that no reading closes is decided once every range of the pass is known instead. const first = readings.reduce(laterClosing); @@ -1480,10 +1493,12 @@ function redactSecretShapedWalk( * Render any result to the text the model would see (a string is itself; else compact JSON) and to the text * its summary is made from. Inline media bytes are redacted first so the summary/spill/preview can never * carry base64 (I3). A non-string result's summary text is its JSON with every member under a - * credential-named key and every argument after a credential flag masked whole (`structuredSummaryText`), - * which no text rule reads. The text rules read the JSON as one text (`makeSummary`), a value one JSON - * string down included, so a PEM block or a quoted value that runs across members keeps the context it is - * read in; projecting each string alone would mask a key or a PEM marker that a later member's text needs. + * credential-named key and every argument after a credential flag masked whole (`structuredSummaryText`): + * the text rules read those values once in the unwalked JSON, with the whole masks as ranges of that same + * pass, and the later passes read only the masks. The text rules read the JSON as one text (`makeSummary`), + * a value one JSON string down included, so a PEM block or a quoted value that runs across members keeps + * the context it is read in; projecting each string alone would mask a key or a PEM marker that a later + * member's text needs. */ function toText(result: unknown): { readonly text: string; readonly summaryText: string } { if (typeof result === 'string') { diff --git a/tools/causal-controls/provenance.test.mjs b/tools/causal-controls/provenance.test.mjs index 06452041..f31a11e7 100644 --- a/tools/causal-controls/provenance.test.mjs +++ b/tools/causal-controls/provenance.test.mjs @@ -358,7 +358,7 @@ test('secrets workload corrections retain independent complete-count and reader- // Sixth pre-review and round 9: a summary's escaped separators and its unwalked reading. 'core-summary-escape-view', 'core-summary-pem-markers', - ].map((id) => [id, ['packages/core/src/tools/bounding.test.ts'], 217]), + ].map((id) => [id, ['packages/core/src/tools/bounding.test.ts'], 218]), ['core-unsettled-refusal', ['packages/core/src/tools/registry.test.ts'], 115], ...[ 'diagnostic-joined-before-masking', diff --git a/tools/secrets-controls/README.md b/tools/secrets-controls/README.md index 96898776..516eba67 100644 --- a/tools/secrets-controls/README.md +++ b/tools/secrets-controls/README.md @@ -178,7 +178,7 @@ modern `sk-` body characters, joined-span application, the original-text left-bo check, split scheme labels, split API-header labels and raw `?`/`#` password recognition independently. Four LLM controls remove the before-masking LF/TAB probe, its original left-boundary check, split API-header recognition and raw-password recognition. -Every core projection phase executes the complete 217-case bounding file (the journal-identity +Every core projection phase executes the complete 218-case bounding file (the journal-identity control's phases the complete 115-case registry file); every new LLM phase executes the complete 224-case scrubber file. Independent ordinary tooling tests pin these inventories, the 278-case overlap cohort, 180-case CLI diagnostic cohort and 18-case command-export cohort. Actual collection still has to match in every phase. @@ -261,7 +261,7 @@ previous-word prefilter and the rule's own boundary predicate; retaining either would leave the tested policy active. Header/key recovery removals still change their original-rule boundary only, and all removals retain the same canonical matchers and bounded raw/joined standalone visitors. Current collection is 26 -complete files / 1,647 cases, with each control's exact workload and named target +complete files / 1,648 cases, with each control's exact workload and named target checked separately. These are updated declarations and author checks, not new physical causal captures or a clean review round. R3 is preserved as NONCLEAN; later corrective acceptance belongs to the owning phase register. Prior selected @@ -270,8 +270,8 @@ later corrective acceptance belongs to the owning phase register. Prior selected The current 130-control / 390-phase full declaration runs as two round-robin shards of 65 controls (`--shard 1/2` and `--shard 2/2`), each exporting its own evidence: each projects 1,177 retained files (65 × 18 + 7) of the unchanged 2,000. Locally both exports held exactly that many and measured -50.19 MiB and 50.09 MiB, final export record included, against the unchanged -64-MiB bound. That is one local measurement at the round-9 source, not a CI capture; one export of +50.24 MiB and 50.14 MiB, final export record included, against the unchanged +64-MiB bound. That is one local measurement at the round-10 source, not a CI capture; one export of the whole family would exceed the byte bound. Historical 41-control captures retain their original source and cohort attribution; selected B2 or original-boundary recovery captures are not full-family acceptance. @@ -531,15 +531,30 @@ mutation: the quoted value's plain and scheme views, the gate that keeps a schem a plain reading reaches, and the latest closing; the summary's stopped reading and its later closer; the unclosed value's views with and without the key rule's masks, their kept variants, the guard on a view that fills the value's quote and the latest reread; the unwalked reading's placeholders, -whole and inside a string, a value with no JSON form, its mask and its collision guard; the -scheme-labelled and argument whole masks; the wrapped markers in the summary alone, their whitespace -runs and the prefilter; and the opaque parameter in a URL username. An admitted-size case deadline +whole and inside a string, a value with no JSON form through its `??` fallback, its mask and its +collision guard; the scheme-labelled and argument whole masks; the wrapped markers in the summary +alone, their whitespace between `PRIVATE` and `KEY` and the prefilter; and the opaque parameter in a +URL username. (Corrected after round 10: the no-JSON-form value's try/catch and the whitespace inside +the wrapped markers' type words have no killing test; both gaps are deferred.) An admitted-size case deadline pins the four quadratic shapes in the journal, display and summary paths; a timeout counts as infrastructure, so no control can certify it. Five mutations are output-neutral and have neither a test nor a control: the closing mask of a summary value that also stops at a line break, which its later closer covers; a kept run's clamp to its range, whose empty ranges a fill skips anyway; and three shortcuts that keep or reuse a view when no mask ends with a backslash. +The tenth round, the last under the maintainer's round cap, found no high finding. The maintainer +counted one realistic gap that base shares as high: in a structured result's JSON a query value at +the end of a line runs on through the next line's key and takes the backslash of its value's escaped +quote, so a `cat .env` result's password showed in its summary. A summary now opens such a value +where the earlier mask keeps its trailing backslashes; the journal keeps its reading. +`core-quoted-value-scan` and `core-escaped-quote` are re-anchored on the changed lines; no control is +added. Pinned by a test without a control, each killed by its mutation: the summary-only gate, the +kept opening and both of its reading sets. Two mutations are output-neutral and have neither a test +nor a control: the condition that the filled view holds no backslash at the opening and the +condition that the kept view holds one, since a mask the kept view keeps ends before a delimiter, +where a view that keeps the backslashes closes no later than the filled one. The round's other +findings are recorded in deferred-tasks.md. + The thin entry point uses the same coordinator, outcome and bounded process join as the filesystem family in [causal-controls](../causal-controls/README.md). The provider-header diagnostic control pins the store's fixed throwable message diff --git a/tools/secrets-controls/mutations.mjs b/tools/secrets-controls/mutations.mjs index 41bf273c..7402eed2 100644 --- a/tools/secrets-controls/mutations.mjs +++ b/tools/secrets-controls/mutations.mjs @@ -334,7 +334,7 @@ const withoutCoreOriginalBoundary = coreOriginalBoundary ) .replace(" if (boundary?.test(text[originalStart - 1] ?? '')) return;", ''); /** The complete core projection test file every core projection control runs. */ -const CORE_PROJECTION_CASES = 217; +const CORE_PROJECTION_CASES = 218; function coreProjectionControl(id, before, after, targets, marker = coreProjectionMarker) { return control( id, @@ -1553,12 +1553,13 @@ export const mutations = [ ), coreProjectionControl( 'core-quoted-value-scan', - ' const value = quotedValueEnd(values, valueStart);', + ' const readings = kept === undefined ? [quotedValueEnd(values, valueStart), ...plain] : plain;', String.raw` const quoted = opening === '"' ? /"(?:[^"\\\r\n]|\\[^\r\n])*"/y : /'(?:[^'\\\r\n]|\\[^\r\n])*'/y; quoted.lastIndex = valueStart; const value = quoted.test(values) ? { end: quoted.lastIndex, closed: true, lineBreak: false } - : { end: valueStart, closed: false, lineBreak: true };`, + : { end: valueStart, closed: false, lineBreak: true }; + const readings = kept === undefined ? [value, ...plain] : plain;`, ['projects exactly admitted-size quoted values without exhausting the RegExp stack'], 'SECRETS_B7_CORE_QUOTED_SCAN', ), @@ -1602,8 +1603,8 @@ export const mutations = [ ), coreProjectionControl( 'core-escaped-quote', - ' const opening = escaped ? values[valueStart + 1] : values[valueStart];', - ' const opening = values[valueStart];', + ' const opening = escaped ? opener[valueStart + 1] : opener[valueStart];', + ' const opening = opener[valueStart];', ['reads a quoted value one JSON string down'], 'SECRETS_B8_CORE_ESCAPED_QUOTE', ), From 2a1f7359ab8d220f8b53b132d1c4c95f6c6f6c05 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Thu, 22 Oct 2026 03:13:04 +0300 Subject: [PATCH 20/97] test(core): split the round-9 linear-views case for the coverage job CI's engine coverage floor job failed on 9eb38156: round 9's admitted-size case 'keeps every projection linear where masks share a run of backslashes or end in one' ran ten 8-MiB projections in one case. It takes 3.7 s plain and 7.2 s under local V8 coverage, and on the GitHub runner it passed the 15 s case deadline (bounding.test.ts:2964, "Test timed out in 15000ms"). The case is now one case per shape (three input shapes and the structured summary), with the same inputs, assertions and deadline; each takes 1.1-2.8 s under local coverage. No deadline is extended. The core projection cases go from 218 to 221, so CORE_PROJECTION_CASES, the provenance pin and the secrets controls README follow. The local coverage gate passes: llm 97.79 % lines / 95.69 % branches, mcp 96.56 / 91.29, core (measured) 96.71 / 91.06. Refs: ADR-0113 Co-Authored-By: Claude Opus 5.5 --- packages/core/src/tools/bounding.test.ts | 44 +++++++++++++++-------- tools/causal-controls/provenance.test.mjs | 2 +- tools/secrets-controls/README.md | 8 +++-- tools/secrets-controls/mutations.mjs | 2 +- 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/packages/core/src/tools/bounding.test.ts b/packages/core/src/tools/bounding.test.ts index 2bd6194b..6a7e4559 100644 --- a/packages/core/src/tools/bounding.test.ts +++ b/packages/core/src/tools/bounding.test.ts @@ -2961,19 +2961,28 @@ describe('PR95 review round 9: closing views, unwalked members, wrapped markers ).toBe('https://[redacted]@host.example/x?q=1'); }); - it( - 'keeps every projection linear where masks share a run of backslashes or end in one', - async () => { - // Each view of a quoted value is built once per pass, and a run of backslashes that many masks - // end in is read once. - for (const [input, summary] of [ - [`password="x\n${'token='.repeat(1_000_000)}${'\\'.repeat(1_000_000)} end`, '[redacted]'], - [`${'password="abcdefgh\n'.repeat(400_000)}token=abcdef\\ end`, '[redacted]'], - [ - `${'"password": "abc", '.repeat(400_000)}\\"https://h.example/x?sig=${'a'.repeat(30)}\\" end`, - '"[redacted], "[redacted],', - ], - ] as const) { + // Each view of a quoted value is built once per pass, and a run of backslashes that many masks end in + // is read once. One shape per case, so that each keeps the case deadline under coverage instrumentation. + for (const [shape, input, summary] of [ + [ + 'nested keys share a run of backslashes', + `password="x\n${'token='.repeat(1_000_000)}${'\\'.repeat(1_000_000)} end`, + '[redacted]', + ], + [ + 'many values stop at a line break', + `${'password="abcdefgh\n'.repeat(400_000)}token=abcdef\\ end`, + '[redacted]', + ], + [ + 'many values precede a mask that ends in a backslash', + `${'"password": "abc", '.repeat(400_000)}\\"https://h.example/x?sig=${'a'.repeat(30)}\\" end`, + '"[redacted], "[redacted],', + ], + ] as const) { + it( + `keeps every projection linear where ${shape}`, + async () => { expect(redactSecretShapedText(input).endsWith(' end'), 'SECRETS_B9_CORE_LINEAR_VIEWS').toBe( true, ); @@ -2985,7 +2994,14 @@ describe('PR95 review round 9: closing views, unwalked members, wrapped markers (await boundForModel(input, BIG, {})).summary.startsWith(summary), 'SECRETS_B9_CORE_LINEAR_VIEWS', ).toBe(true); - } + }, + ADMITTED_SIZE_CASE_TIMEOUT_MS, + ); + } + + it( + "keeps a structured summary linear where a value's later closer is far away", + async () => { const structured = { stdout: `password="first\nsecond" ${'log line ok\n'.repeat(600_000)}echo "token=abcdef12"\n`, }; diff --git a/tools/causal-controls/provenance.test.mjs b/tools/causal-controls/provenance.test.mjs index f31a11e7..3ef0613b 100644 --- a/tools/causal-controls/provenance.test.mjs +++ b/tools/causal-controls/provenance.test.mjs @@ -358,7 +358,7 @@ test('secrets workload corrections retain independent complete-count and reader- // Sixth pre-review and round 9: a summary's escaped separators and its unwalked reading. 'core-summary-escape-view', 'core-summary-pem-markers', - ].map((id) => [id, ['packages/core/src/tools/bounding.test.ts'], 218]), + ].map((id) => [id, ['packages/core/src/tools/bounding.test.ts'], 221]), ['core-unsettled-refusal', ['packages/core/src/tools/registry.test.ts'], 115], ...[ 'diagnostic-joined-before-masking', diff --git a/tools/secrets-controls/README.md b/tools/secrets-controls/README.md index 516eba67..6cdd960a 100644 --- a/tools/secrets-controls/README.md +++ b/tools/secrets-controls/README.md @@ -178,7 +178,7 @@ modern `sk-` body characters, joined-span application, the original-text left-bo check, split scheme labels, split API-header labels and raw `?`/`#` password recognition independently. Four LLM controls remove the before-masking LF/TAB probe, its original left-boundary check, split API-header recognition and raw-password recognition. -Every core projection phase executes the complete 218-case bounding file (the journal-identity +Every core projection phase executes the complete 221-case bounding file (the journal-identity control's phases the complete 115-case registry file); every new LLM phase executes the complete 224-case scrubber file. Independent ordinary tooling tests pin these inventories, the 278-case overlap cohort, 180-case CLI diagnostic cohort and 18-case command-export cohort. Actual collection still has to match in every phase. @@ -261,7 +261,7 @@ previous-word prefilter and the rule's own boundary predicate; retaining either would leave the tested policy active. Header/key recovery removals still change their original-rule boundary only, and all removals retain the same canonical matchers and bounded raw/joined standalone visitors. Current collection is 26 -complete files / 1,648 cases, with each control's exact workload and named target +complete files / 1,651 cases, with each control's exact workload and named target checked separately. These are updated declarations and author checks, not new physical causal captures or a clean review round. R3 is preserved as NONCLEAN; later corrective acceptance belongs to the owning phase register. Prior selected @@ -553,7 +553,9 @@ kept opening and both of its reading sets. Two mutations are output-neutral and nor a control: the condition that the filled view holds no backslash at the opening and the condition that the kept view holds one, since a mask the kept view keeps ends before a delimiter, where a view that keeps the backslashes closes no later than the filled one. The round's other -findings are recorded in deferred-tasks.md. +findings are recorded in deferred-tasks.md. After the closure, CI's coverage job took round 9's +linear-views case past the case deadline, since instrumented coverage slows every projection, so that +case became one case per shape (four cases, the same inputs and assertions). The thin entry point uses the same coordinator, outcome and bounded process join as the filesystem family in [causal-controls](../causal-controls/README.md). diff --git a/tools/secrets-controls/mutations.mjs b/tools/secrets-controls/mutations.mjs index 7402eed2..d164c289 100644 --- a/tools/secrets-controls/mutations.mjs +++ b/tools/secrets-controls/mutations.mjs @@ -334,7 +334,7 @@ const withoutCoreOriginalBoundary = coreOriginalBoundary ) .replace(" if (boundary?.test(text[originalStart - 1] ?? '')) return;", ''); /** The complete core projection test file every core projection control runs. */ -const CORE_PROJECTION_CASES = 218; +const CORE_PROJECTION_CASES = 221; function coreProjectionControl(id, before, after, targets, marker = coreProjectionMarker) { return control( id, From 4c3cafe951003dc9ea3eaa871677882b0ca75e44 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Thu, 22 Oct 2026 04:34:32 +0300 Subject: [PATCH 21/97] ci(repo): shard the filesystem causal controls under the evidence bound MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI's filesystem causal controls failed on 2a1f7359 at export with "Raw phase manifest bound" (tools/causal-controls/retention.mjs:400), and the lint · typecheck · test job's guard then refused its result. Every phase retains an exact manifest of the whole source graph; once Wave 3's PR95 increment grew that graph, a manifest measured 203,421 bytes and the 330 manifests of one export 67,128,930 bytes, 20,066 over the unchanged 64-MiB retained-evidence bound. The family's capacity arithmetic had counted files only (1,987 of 2,000). As the secrets family did in PR95's round 7, the filesystem job now runs two legs that each run and export a disjoint round-robin half (`--shard k/2`, fail-fast off, one artifact per leg) under the unchanged per-phase, case, output, 2,000-file and 64-MiB bounds. Both shards ran locally with the export at 2a1f7359's source: 55 controls each, all passing; 165 manifests (33,564,465 bytes) and 997 files per export, 25.65 MiB and 25.59 MiB. require-w8-acceptance.test.mjs now checks both families' sharded jobs (an unsharded filesystem command fails it), and the filesystem controls README records the bound and the measurement. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 14 ++++++++++--- tools/ci/require-w8-acceptance.test.mjs | 28 ++++++++++--------------- tools/fs-policy-controls/README.md | 12 +++++++++++ 3 files changed, 34 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7005f1d1..1ef328ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -245,13 +245,21 @@ jobs: # Offline, uncached semantic removal/restoration evidence. Required indirectly through ci.needs. # ADR-0105 isolated, current-source removal/restoration evidence, required through ci.needs. fs-mutations: - name: Wave 2 filesystem causal controls + name: Wave 2 filesystem causal controls (shard ${{ matrix.shard }} of 2) # ADR-0106/0109 extend the original 26 controls with complete media/lifetime workloads. # The 20-minute job historically reached only 46 of 93 controls; the later budget covered 99. # The current cohort is 110 controls (2026-10-15); this comment does not certify its execution. # Per-phase 90-second/test 15-second deadlines and output/evidence ceilings remain unchanged. + # Every phase retains a manifest of the whole source graph (about 200 KB each), so the 330 phase + # manifests of one export passed the unchanged 64-MiB retained-evidence bound once Wave 3's PR95 + # increment grew that graph (2026-10-22). As for the secrets controls, two legs each run and + # export a disjoint round-robin half (`--shard k/2`) under the unchanged bounds. timeout-minutes: 60 runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: [1, 2] steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: @@ -268,12 +276,12 @@ jobs: - name: Run filesystem removal and restoration controls env: RELAVIUM_ACCEPTANCE_EVIDENCE: ${{ runner.temp }}/wave2-acceptance - run: pnpm check:fs-mutations + run: pnpm check:fs-mutations --shard ${{ matrix.shard }}/2 - name: Retain bounded filesystem control metadata if: ${{ always() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: wave2-filesystem-${{ github.run_id }}-${{ github.run_attempt }} + name: wave2-filesystem-${{ matrix.shard }}-of-2-${{ github.run_id }}-${{ github.run_attempt }} path: ${{ runner.temp }}/wave2-acceptance/filesystem if-no-files-found: warn retention-days: 7 diff --git a/tools/ci/require-w8-acceptance.test.mjs b/tools/ci/require-w8-acceptance.test.mjs index 0365f319..d33e642e 100644 --- a/tools/ci/require-w8-acceptance.test.mjs +++ b/tools/ci/require-w8-acceptance.test.mjs @@ -89,7 +89,12 @@ test('the actual required workflow joins both full offline controls and checks t ); assert.equal(secretsGuard.if, undefined); assert.equal(secretsGuard['continue-on-error'], undefined); - for (const kind of ['secrets']) { + // Both families run as two legs that each run and export a disjoint half under the unchanged + // 64-MiB retained-evidence bound, which one export of either whole cohort no longer fits. + for (const [kind, directory] of [ + ['fs', 'filesystem'], + ['secrets', 'secrets'], + ]) { const job = workflow.jobs[`${kind}-mutations`]; assert.equal(job['continue-on-error'], undefined); assert.equal(job.if, undefined); @@ -107,24 +112,13 @@ test('the actual required workflow joins both full offline controls and checks t assert.equal(command.env.RELAVIUM_ACCEPTANCE_EVIDENCE, '${{ runner.temp }}/wave2-acceptance'); const artifact = job.steps.find((step) => step.uses?.startsWith('actions/upload-artifact@')); assert.equal(artifact.if, '${{ always() }}'); - assert.equal(artifact.with.path, '${{ runner.temp }}/wave2-acceptance/secrets'); - assert.match(artifact.with.name, /^wave2-secrets-\$\{\{ matrix\.shard \}\}-of-2-/); + assert.equal(artifact.with.path, `\${{ runner.temp }}/wave2-acceptance/${directory}`); + assert.ok( + artifact.with.name.startsWith(`wave2-${directory}-\${{ matrix.shard }}-of-2-`), + artifact.with.name, + ); assert.equal(artifact.with['retention-days'], 7); } - const filesystem = workflow.jobs['fs-mutations']; - assert.equal(filesystem['continue-on-error'], undefined); - assert.equal(filesystem.if, undefined); - assert.equal(filesystem['timeout-minutes'], 60); - assert.ok(filesystem.steps.some((step) => step.run === 'pnpm install --frozen-lockfile')); - const fsCommand = filesystem.steps.find((step) => step.run === 'pnpm check:fs-mutations'); - assert.equal(fsCommand['continue-on-error'], undefined); - assert.equal(fsCommand.env.RELAVIUM_ACCEPTANCE_EVIDENCE, '${{ runner.temp }}/wave2-acceptance'); - const fsArtifact = filesystem.steps.find((step) => - step.uses?.startsWith('actions/upload-artifact@'), - ); - assert.equal(fsArtifact.if, '${{ always() }}'); - assert.equal(fsArtifact.with.path, '${{ runner.temp }}/wave2-acceptance/filesystem'); - assert.equal(fsArtifact.with['retention-days'], 7); const packageJson = JSON.parse( readFileSync(new URL('../../package.json', import.meta.url), 'utf8'), ); diff --git a/tools/fs-policy-controls/README.md b/tools/fs-policy-controls/README.md index b9a9f965..60b8a41a 100644 --- a/tools/fs-policy-controls/README.md +++ b/tools/fs-policy-controls/README.md @@ -181,6 +181,18 @@ projecting 1,987 retained files (110 × 18 + 7) under the unchanged 2,000-file /64-MiB ceilings. This is capacity arithmetic, not a measured export or completed capture; historical captures retain their own source heads and actual counts. +That arithmetic counted files only. Every phase also retains its exact source manifest, +one per phase and about 200 KB each because it lists the whole source graph, and the +export's raw manifests share the 64-MiB bound. Once Wave 3's PR95 increment grew that +graph, a manifest measured 203,421 bytes and the 330 manifests of one export +67,128,930 bytes, 20,066 over the bound, so CI's export refused (2026-10-22). Like the +secrets family, the declaration now runs as two round-robin shards of 55 controls +(`--shard 1/2` and `--shard 2/2`), each exporting its own evidence: 165 manifests +(33,564,465 bytes) and 997 retained files (55 × 18 + 7). Both shards ran locally with +`RELAVIUM_ACCEPTANCE_EVIDENCE` set at `2a1f7359`'s source; every control passed, and +the exports measured 25.65 MiB and 25.59 MiB. That is one local measurement, not a CI +capture. + The declaration is not acceptance evidence by itself. The complete matrix still requires a stable committed head, all fixed/removal/restored phases, named semantic witnesses, child source/dependency receipts, byte-exact restoration, From a72c7ecd878ea34238f855d88835fdd85ca56220 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Thu, 22 Oct 2026 06:01:05 +0300 Subject: [PATCH 22/97] feat(repo): build and finalize the CLI release candidate (Step 9a.1) Wave 3 Step 9a.1, the first increment of the CLI release map/custody substrate (#46, ADR-0111): the candidate build and its prepack finalizer. The receipt stays compiled_package_test; symbolication, the private map manifest, custody and the release state controller remain Steps 9a.2-9d. - Owned candidate tree (tools/cli-package/candidate.mjs): the producer never builds in the developer's workspace. It records the checkout's commit and porcelain status, copies every file Git would ship (by the repository's ignore rules and the developer's global excludes file, which is the only global setting it reads) into a fresh owner-only tree with exact modes, refuses if the commit or status changed during the copy, installs the locked closure offline, frozen and without lifecycle scripts, and verifies that every workspace link resolves inside the tree. - Producer (produce.mjs, run by pack.mjs): fresh uncached upstream tsc maps for the five engine packages, the sole prepack build, a build cache check before the seal, owner-only staging outside the artifact, and removal of the private root on every path; its composition is tested through injected ports. - Prepack (prepack.mjs) runs tsup once; with a validated staging directory the candidate-mode build emits external maps without sourcesContent and the finalizer (finalize.mjs) accounts for every output, validates every upstream and final map, rewrites sources to repository-relative original TypeScript, removes only the exact map trailers before hashing and stages maps and metafile privately. - The seal reads the candidate and its staging explicitly; the closure guard is map-aware and refuses any map or map link in an ordinary build; the PR #95 fold-ins remove the archive's dead TAR_FORMAT branches and replace the installed smoke's ancestor loop. - release.yml's publish job runs a fail-closed guard first: its entry point exits 1 before it evaluates its rule, so no artifact and no way of invoking it can reach npm publish until Step 9d's custody exists (the Interim release guard ruling of 2026-10-19). An internal pre-review of the prepared change found seventeen issues, none high once verified; all are corrected here (the guard's entry point, the tested producer, the global excludes, exact modes and the test gaps). A mutation run of twenty mutants over the new code kills all of them. The full local CI then caught a top-level await in closure.test.mjs that ran after most tests were registered, so a slow fixture build let the runner remove the file's scratch directory before a later test; both fixtures are now built before the first test. Docs: packaging.md (Status and the 9a.1 implementation boundary), the release runbook, an ADR-0051 amendment, the tooling README, the phase register (9a.1 row and checkpoint, and the PR95 closure's post-closure CI corrections 2a1f7359 and 4c3cafe9), deferred-tasks.md (the PR #95 tooling nits) and the status lines. Refs: ADR-0111, ADR-0051 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 31 +- AGENTS.md | 2 +- CLAUDE.md | 2 +- apps/cli/package.json | 2 +- apps/cli/tsup.config.ts | 16 +- ...distribution-thin-bundle-private-engine.md | 2 + docs/reference/cli/packaging.md | 61 +- docs/roadmap/current.md | 2 +- docs/roadmap/deferred-tasks.md | 2 +- .../phase-2.5.5-hardening-and-remediation.md | 64 +- docs/runbooks/release-a-surface.md | 8 + package.json | 2 +- tools/bundle-closure/check.mjs | 39 +- tools/bundle-closure/closure.mjs | 60 +- tools/bundle-closure/closure.test.mjs | 165 ++++- tools/bundle-closure/inventory.mjs | 47 ++ tools/ci/require-packed-cli.test.mjs | 30 +- tools/cli-package/README.md | 27 +- tools/cli-package/archive.mjs | 2 - tools/cli-package/candidate.mjs | 338 ++++++++++ tools/cli-package/candidate.test.mjs | 578 ++++++++++++++++++ tools/cli-package/check.mjs | 20 +- tools/cli-package/finalize.mjs | 462 ++++++++++++++ tools/cli-package/finalize.test.mjs | 478 +++++++++++++++ tools/cli-package/pack.mjs | 59 +- tools/cli-package/pipeline.mjs | 50 +- tools/cli-package/pipeline.test.mjs | 60 +- tools/cli-package/prepack.mjs | 62 ++ tools/cli-package/prepack.test.mjs | 133 ++++ tools/cli-package/produce.mjs | 98 +++ tools/cli-package/produce.test.mjs | 179 ++++++ tools/cli-package/release-guard-core.mjs | 45 ++ tools/cli-package/release-guard.mjs | 26 + tools/cli-package/release-guard.test.mjs | 111 ++++ tools/cli-package/seal.mjs | 79 +-- tools/cli-package/seal.test.mjs | 71 ++- tools/cli-package/smoke/installation.mjs | 19 + tools/cli-package/smoke/installation.test.mjs | 43 ++ tools/cli-package/smoke/worker.mjs | 5 +- 39 files changed, 3305 insertions(+), 175 deletions(-) create mode 100644 tools/bundle-closure/inventory.mjs create mode 100644 tools/cli-package/candidate.mjs create mode 100644 tools/cli-package/candidate.test.mjs create mode 100644 tools/cli-package/finalize.mjs create mode 100644 tools/cli-package/finalize.test.mjs create mode 100644 tools/cli-package/prepack.mjs create mode 100644 tools/cli-package/prepack.test.mjs create mode 100644 tools/cli-package/produce.mjs create mode 100644 tools/cli-package/produce.test.mjs create mode 100644 tools/cli-package/release-guard-core.mjs create mode 100644 tools/cli-package/release-guard.mjs create mode 100644 tools/cli-package/release-guard.test.mjs create mode 100644 tools/cli-package/smoke/installation.mjs create mode 100644 tools/cli-package/smoke/installation.test.mjs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 83c0ea3e..8eafc2a3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,6 +3,13 @@ # Step 7b reuses the required installed-package runner, including the exact Node 22.0.0 runtime floor. # Step 9 retains ownership of original-TS maps, release custody and their permission/discovery redesign. # +# PUBLICATION IS REFUSED until that custody exists (Wave 3 Step 9a.1, the Interim release guard ruling of +# 2026-10-19): `publish` first runs a fail-closed guard that refuses every artifact. Only a verified release +# candidate (exact tarball, original ciphertext and release binding receipt) may publish, and Step 9d's custody +# validator, which replaces the guard, is what will verify one; Wave 4b supplies the production recipient. Until +# both exist every tag push stops at the guard, after the cross-OS smoke, so not even a v0.1.x patch publishes. +# The flow description below is the pre-custody shape the guard now blocks. +# # Flow: `pack` builds the engine-inlined bundle and `pnpm pack`s it (resolving catalog:/workspace: to # concrete versions) into a tarball artifact → `smoke` installs THAT SAME tarball outside the workspace on ubuntu / # macOS / Windows and the minimum Node runtime and asserts the binary works (prebuilt native addons load, the bundled engine runs) → @@ -92,8 +99,9 @@ jobs: cache: pnpm - name: Install (frozen lockfile) run: pnpm install --frozen-lockfile - # The same required package runner owns fresh dependencies and the sole prepack CLI build. - # Original-TS maps and encrypted release custody are staged separately in Wave 3 Step 9. + # The same required package runner owns fresh dependencies and the sole prepack CLI build in an owned + # candidate tree, where original-TS maps are built, validated, staged privately and discarded (Wave 3 + # Step 9a.1); their encrypted release custody is Steps 9b–9d. - name: Pack once and independently inspect the public package env: RELAVIUM_PACKAGE_SOURCE_SHA: ${{ github.sha }} @@ -215,13 +223,18 @@ jobs: contents: read id-token: write # npm provenance attestation steps: - # Checkout ONLY `.nvmrc` (sparse, non-cone) so `setup-node` can read the single Node-version source the - # pack/smoke jobs use too; publish needs no other repo files — it pushes the downloaded tarball, not a - # fresh build. (fetch-depth defaults to 1, so it is not set again.) + # Checkout ONLY `.nvmrc` and the dependency-free publication guard, its entry point and its rule (sparse, + # non-cone): `setup-node` reads the single Node-version source the pack/smoke jobs use too, and the guard + # runs before any publish. + # Publish needs no other repo files — it pushes the downloaded tarball, not a fresh build. + # (fetch-depth defaults to 1, so it is not set again.) - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: persist-credentials: false - sparse-checkout: .nvmrc + sparse-checkout: | + .nvmrc + tools/cli-package/release-guard.mjs + tools/cli-package/release-guard-core.mjs sparse-checkout-cone-mode: false - name: Set up Node (npm registry) uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 @@ -233,6 +246,12 @@ jobs: with: name: ${{ needs.packed-cli.outputs.artifact-name }} path: artifact + # Interim fail-closed guard (Wave 3 Step 9a.1, the Interim release guard ruling of 2026-10-19): only a + # verified release candidate may publish, and none can exist before Step 9d's custody graph and Wave 4b's + # production recipient. Every artifact, including this package-test tarball, therefore stops here; the + # entry point exits 1 before it evaluates anything, so no way of invoking it can skip the refusal. + - name: Refuse anything but a verified release candidate + run: node tools/cli-package/release-guard.mjs ./artifact # Publish the EXACT artifact the smoke matrix proved — not a fresh build. catalog:/workspace: are already # resolved in the packed manifest, so `npm publish ` is correct here (pnpm-pack did the resolving). - name: Publish diff --git a/AGENTS.md b/AGENTS.md index 5272df09..ba694a3a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -129,7 +129,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Next is Step 9a.1 (release map/custody substrate, #46/ADR-0111). `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (release map/custody substrate, #46/ADR-0111) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-implementation-checkpoint--2026-10-22)). `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. See [docs/roadmap/current.md](docs/roadmap/current.md) for live status. diff --git a/CLAUDE.md b/CLAUDE.md index 0fa4df88..a0f0c9ff 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -218,7 +218,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Next is Step 9a.1 (release map/custody substrate, #46/ADR-0111). `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (release map/custody substrate, #46/ADR-0111) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-implementation-checkpoint--2026-10-22)). `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. For live status, per-PR history, milestone dates, and open obligations, see the canonical diff --git a/apps/cli/package.json b/apps/cli/package.json index 101ab679..e6a9cfce 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -39,7 +39,7 @@ }, "scripts": { "build": "tsup", - "prepack": "tsup && pnpm -w run lint:bundle-closure", + "prepack": "node ../../tools/cli-package/prepack.mjs", "typecheck": "tsc -p tsconfig.json --noEmit", "lint": "eslint src", "test": "vitest run" diff --git a/apps/cli/tsup.config.ts b/apps/cli/tsup.config.ts index 92e89c71..2f51c7df 100644 --- a/apps/cli/tsup.config.ts +++ b/apps/cli/tsup.config.ts @@ -17,10 +17,18 @@ const { version } = JSON.parse( * are bundler-hostile). `tools/bundle-closure/check.mjs` guards that this `external` list equals the declared * runtime `dependencies`, so the two can never drift. * - * `sourcemap: false` + `minify: true`: the published `dist/` must ship no source tree or maps. This is the npm + * `minify: true` and no maps in `dist/`: the published package ships no source tree or maps. This is the npm * distribution boundary, not a claim that the public repository's sources are secret (ADR-0111). * The entry-source shebang makes `dist/index.js` directly executable as the `relavium` bin. + * + * Ordinary builds are map-free. A candidate build — this build with `RELAVIUM_CLI_CANDIDATE_STAGING` defined, which + * the prepack lifecycle (`tools/cli-package/prepack.mjs`, Wave 3 Step 9a.1) runs only after it validates that + * directory as private staging — emits external maps without embedded sources. The prepack finalizer + * (`tools/cli-package/finalize.mjs`) then checks that they chain to original TypeScript, moves them and the + * metafile out of the package and removes their link trailers before anything is hashed or packed; the ordinary + * closure guard refuses any map that a build outside prepack leaves in `dist`. */ +const CANDIDATE = process.env.RELAVIUM_CLI_CANDIDATE_STAGING !== undefined; const THIRD_PARTY_EXTERNAL = [ '@anthropic-ai/sdk', '@clack/prompts', @@ -51,7 +59,11 @@ export default defineConfig({ target: 'node22', platform: 'node', clean: true, - sourcemap: false, + sourcemap: CANDIDATE, + esbuildOptions(options) { + // Original sources never enter a candidate map; the finalizer refuses any that does. + if (CANDIDATE) options.sourcesContent = false; + }, minify: true, splitting: true, // Emit esbuild's metafile (`dist/metafile-esm.json`) — the authoritative record of every externalized import, diff --git a/docs/decisions/0051-cli-distribution-thin-bundle-private-engine.md b/docs/decisions/0051-cli-distribution-thin-bundle-private-engine.md index 1df8188f..732ae511 100644 --- a/docs/decisions/0051-cli-distribution-thin-bundle-private-engine.md +++ b/docs/decisions/0051-cli-distribution-thin-bundle-private-engine.md @@ -151,3 +151,5 @@ intended precedent for the desktop (`.dmg`) and VS Code (`.vsix`) surfaces. ## Wave 3 decision acceptance — 2026-10-16 > Amended 2026-10-16: [ADR-0111](0111-cli-meta-operations-use-a-lazy-packaged-runtime.md) is accepted by the maintainer. Only the single-output constraint is partially replaced by finite engine-inlined runtime chunks with complete closure and packed-platform checks. Licensing rather than source secrecy, private engine packages and no public plaintext source maps remain accepted. Exact release-map encrypted custody is a distribution policy, not secrecy for this public source repository. The approved [execution plan](../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) owns implementation and review gates; this note does not claim implementation landing. + +> Amended 2026-10-22: Wave 3 Step 9a.1 implements the candidate build. A candidate build — the CLI build with `RELAVIUM_CLI_CANDIDATE_STAGING` defined, which the prepack lifecycle runs only after it validates that directory as private staging — emits external maps without embedded sources, and its finalizer stages them outside the package before anything is hashed or packed, so the published artifact still carries no map or map link; the finalizer, the closure guard and the archive inspector each refuse one. "The publish build sets `sourcemap: false`" now describes ordinary builds. The [packaging contract](../reference/cli/packaging.md#candidate-build-and-finalizer-implementation-boundary--step-9a1) records the implemented boundary; this note claims no symbolication, custody or release. diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 3df41142..6cc9618a 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -1,6 +1,6 @@ # CLI Packaging Contract -- **Status**: Compiled graph, installed package and Step 8 native activation independently accepted on `development`; Step 9 map/custody and release gates remain staged +- **Status**: Compiled graph, installed package and Step 8 native activation independently accepted on `development`; the Step 9a.1 candidate build and finalizer are implemented, with independent acceptance pending; the remaining Step 9 map/custody and release gates remain staged - **Date**: 2026-10-15 - **Related**: [ADR-0111](../../decisions/0111-cli-meta-operations-use-a-lazy-packaged-runtime.md), [commands.md](commands.md), [release runbook](../../runbooks/release-a-surface.md) @@ -229,6 +229,65 @@ candidate for that version refuses instead of silently superseding it. The finalized JS/map receipt is sealed locally before encryption; the binding receipt is sealed after ciphertext exists. These are distinct, named artifacts. +### Candidate build and finalizer implementation boundary — Step 9a.1 + +Step 9a.1 implements items 1–3 above and the producer half of item 4. This section +describes the implemented mechanism; independent acceptance is recorded in the +[Wave 3 register](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments). + +- **Owned candidate tree.** `pnpm package:cli:pack` never builds in the developer's + workspace. It records the original checkout's commit and full porcelain status, copies + every file Git would ship (tracked plus untracked-but-not-ignored, minus deletions, by + the repository's ignore rules and the developer's global excludes file) into a fresh + owner-only tree, and refuses if that commit or status changed during the copy. + Ignored output, caches, dependencies and private notes never enter it. The locked + closure is installed there offline, frozen and with `--ignore-scripts` from the + existing content-addressed store, discovered with pnpm's store-location variables + only, and every `@relavium/*` link of the CLI and the five inlined engine packages must + resolve inside the tree. +- **Fresh upstream maps.** The five engine packages compile in order with the installed + compiler's `--sourceMap true --inlineSources false --incremental false` overrides after + their `dist` and incremental files are removed. No Turbo or other build cache is + invoked, and a Turbo directory, `*.tsbuildinfo` or `node_modules/.cache` left in the + tree refuses before the seal. Ordinary builds keep their explicit `sourceMap: false`. +- **Sole prepack build.** `prepack` is the tooling orchestrator + `tools/cli-package/prepack.mjs`, which runs the installed tsup once. Without a staging + directory it then runs the ordinary closure guard, as before. The CLI configuration + emits external maps with `sourcesContent` disabled whenever + `RELAVIUM_CLI_CANDIDATE_STAGING` is defined; prepack first refuses unless it names an + absolute, empty, owner-only directory outside the candidate, and then runs the finalizer + in place of the ordinary guard. That guard refuses any map a build outside prepack + leaves in `dist`. +- **Finalizer.** Before any write it accounts for every emitted file (each runtime JS + output paired with exactly one map, plus the metafile, nothing else). It validates every + consumed upstream tsc map: exact terminal map link, version 3, matching `file`, empty + `sourceRoot`, no embedded sources, and only that package's existing original + TypeScript (a type-only module may have empty mappings). It validates every final map: + version 3, no embedded sources or unreviewed fields, and sources resolving to existing + `apps/cli/src` or engine `src` TypeScript that equal exactly the original sources of + the inputs contributing bytes to that output, so a bundler fallback to compiled JS + refuses. A code-splitting facade with no contributing input must have an empty map. + It removes only tsup's exact terminal `//# sourceMappingURL=.map` trailer and + requires the remaining bytes to equal the size esbuild recorded. Then it stages the + rewritten maps (repository-relative sources; mappings and names unchanged) and the + metafile owner-only outside the package, re-runs the finite closure and the + migration-copy check on the finalized files, and only then hashes into a private record. +- **Seal.** The seal takes the candidate root, the staging directory and the original + checkout's identity explicitly and never resolves a repository from its own location. + Public JS must be exactly the staged metafile's runtime outputs, byte-identical to the + finalizer's record; the actual tarball is inspected as in Step 7b. The receipt remains + `compiled_package_test`. +- **Ordinary map-free control.** The ordinary closure guard refuses any map, map link or + unreviewed file in `dist`, and the five build configurations and the CLI configuration + are pinned map-free outside a named candidate. + +Private maps, the metafile and the hash record are discarded on every path after the +seal; no secure erasure is claimed. The private map manifest, actual packed CLI and engine +symbolication, `check:map-controls` and the `package_test` receipt flip are Step 9a.2; +recipient custody, the release state controller and the workflow graph are Steps 9b–9d. +Until those and Wave 4b's production recipient exist, the release workflow's publish job +refuses every artifact (the [Interim release guard ruling](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) of 2026-10-19). + ## Recipient custody The maintainer selected encrypted CI archival. GnuPG is release tooling, never a diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index b0a9a7e3..ee794f6b 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -171,7 +171,7 @@ bar and round cap, as its medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, by the author alone). Its other findings are recorded in -[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1** follows, the first increment of the release map/custody substrate (#46, ADR-0111), whose +[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is implemented on `development` and under review ([checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-implementation-checkpoint--2026-10-22)); its 9a.1–9d increments carry a dedicated archive/crypto security sitting. Steps 10–19 follow the [increment table](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) and end with whole-wave acceptance at Step 19. `development` is pushed after each accepted increment; PR #95 no longer bounds the diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index f0c6b1ef..1437bca2 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -434,7 +434,7 @@ measured or counted here. `apps/cli/src/harness/concurrency.e2e.test.ts:434-441` uses a fixed 100 ms contention margin; `tools/cli-package/consumer-lifetime.test.mjs:54-59` pre-marks every synthetic case `passed: true` (by design; real detection is the packed-consumer CI). *(nit · the files above)* -- [ ] **Scheduled → Wave 3: the two tooling nits → 9a.1; the duplicate headings → 19b. Tooling and documentation nits.** *Nit, (reviewer).* *Premise corrected 2026-10-19:* the repeated headings are the 2.5.5.A–I sub-stream headings and the Wave 2/Wave 3 "Decisions presented together before implementation" pair, not Wave 2 checkpoint sub-headings. Unreachable `TAR_FORMAT` branches at +- [ ] **Scheduled → Wave 3: the two tooling nits → 9a.1; the duplicate headings → 19b. Tooling and documentation nits.** *Implemented in Wave 3 Step 9a.1 on 2026-10-22, acceptance pending:* the unreachable `TAR_FORMAT` branches are removed, and `installedModulesOwner` (`tools/cli-package/smoke/installation.mjs`) replaces the ancestor loop; the duplicate headings stay with 19b. *Nit, (reviewer).* *Premise corrected 2026-10-19:* the repeated headings are the 2.5.5.A–I sub-stream headings and the Wave 2/Wave 3 "Decisions presented together before implementation" pair, not Wave 2 checkpoint sub-headings. Unreachable `TAR_FORMAT` branches at `tools/cli-package/archive.mjs:126,135`; no `break` in the ancestor loop at `tools/cli-package/smoke/worker.mjs:64-65` (the outermost `node_modules` wins); and repeated heading texts in the phase register, whose later occurrences take a `-1` slug suffix. *(author)* A scan finds repeated H2–H4 headings (the 2.5.5.B–I sub-stream headings and the diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index d5146fa6..18fcc54d 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -3697,7 +3697,7 @@ No credential values or request captures belong in tracked evidence. | PR95 | **Recorded plan amendment, 2026-10-19 (maintainer-approved; new scope from the [PR #95 independent review](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18), not a split of rows 1–19) — scrubber residuals, before 9a:** the `api-key:Bearer`/`Basic` label erasure that leaks the token; raw `?`/`#` URL userinfo masked unless the text before it is host-shaped **and** an `=` assignment separates it from the authority `@`, applied to the LLM leaf and core's twin predicate; an extended finite scheme list (postgres, postgresql, mysql, mariadb, redis, rediss, mongodb, mongodb+srv, amqp, amqps, ssh, sftp, ldap, ldaps, smtp, smtps) masking colon-bearing userinfo for the new schemes; module-load checks of the split-credential scanner's two invariants, a start prefilter, joined-view and glued-scheme witnesses, the load-sensitive 8 MiB timing case, and the causal-control pins. One increment with ordered commits, two NEW clean complete cumulative rounds and a separate security sitting. *Acceptance amended on 2026-10-21 by the maintainer's masker acceptance bar and round cap; closed under the round cap on 2026-10-22 ([checkpoint](#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)).* | | 9 | **Release map/custody substrate:** #46/ADR-0111, independently reviewed after the finite graph. Upstream TS maps, sole prepack final build, pre-hash trailer removal/staging, exact tarball/CLI+engine symbolication, encrypt-once candidate with permissioned prebuild discovery/reservation → pack/reuse → smoke → archive → npm → finalization. Required artifact/platform aggregation, all rerun modes, lost-staging refusal and permission-enforcing offline API controls. Dedicated archive/crypto security sitting. Fixture proof never claims production recipient/custody configuration. | | 9a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage #46/ADR-0111 unchanged, no new contract) — release maps, final build and symbolication:** fresh uncached upstream original-TypeScript maps, the sole prepack final CLI build, final trailer removal before hashes, map and private-graph staging outside public artifacts and caches, exact tarball-to-final-JS binding, and real packed CLI and engine frame symbolication with wrong JS/map/source and missing-map negative controls. Ordinary builds remain map-free. *Split again on 2026-10-19 into 9a.1 and 9a.2 below, which replace it as increments ([Plan shape](#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19)).* | -| 9a.1 | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — candidate build and finalizer:** an owned isolated candidate copy, fresh uncached upstream maps, candidate-mode tsup selection with absent `sourcesContent`, the prepack finalizer (upstream and final map validation, repository-relative sources, exact trailer removal before hashes), owner-only private staging outside public artifacts and caches, closure/seal adaptation, the ordinary map-free control, the PR #95 archive/worker fold-ins, and a fail-closed `release.yml` publish guard that refuses any non-release candidate until 9d and Wave 4b custody exist. The receipt stays `compiled_package_test`. | +| 9a.1 | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — candidate build and finalizer:** an owned isolated candidate copy, fresh uncached upstream maps, candidate-mode tsup selection with absent `sourcesContent`, the prepack finalizer (upstream and final map validation, repository-relative sources, exact trailer removal before hashes), owner-only private staging outside public artifacts and caches, closure/seal adaptation, the ordinary map-free control, the PR #95 archive/worker fold-ins, and a fail-closed `release.yml` publish guard that refuses any non-release candidate until 9d and Wave 4b custody exist. The receipt stays `compiled_package_test`. *Implemented on `development` on 2026-10-22; acceptance pending ([checkpoint](#wave-3-step-9a1-implementation-checkpoint--2026-10-22)).* | | 9a.2 | **Recorded split — manifest, symbolication and package-test receipt:** the private map manifest keyed by `dist/.js`, actual packed CLI and engine frame symbolication against the matching source revision, wrong JS/map/source and missing/malformed-upstream negative controls (product copies via `check:map-controls`, also a required uncached CI job), and the atomic `package_test` receipt flip across seal, binding, consumers, aggregate and both workflows. | | 9b | **Recorded split — recipient custody:** synthetic-only isolated GnuPG controls (public-only home/keyring, pinned primary and encryption-subkey fingerprints, machine-readable refusal of missing, expired, revoked or unusable recipients), encrypt once and reuse the original ciphertext. Explicit crypto/custody security sitting. Fixture proof never claims production recipient or custody configuration. | | 9c | **Recorded split — offline release state controller:** permission-enforcing offline GitHub/npm controller, permissioned prebuild discovery and reservation, and every rerun, partial, conflict and expiry state, with no overwrite, rebuild or re-encryption on uncertain history. | @@ -9510,3 +9510,65 @@ was verified by the author alone, as the round cap provides; every finding it di `development` is pushed and its CI observed, and the PR95 items in [deferred-tasks.md](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18) are closed. Step 9a.1 follows. + +**Post-closure CI correction, 2026-10-22.** `development` was pushed at `9eb38156`, and every CI job +passed except the separate engine coverage floor job, which `pnpm run ci` does not include: under +instrumented coverage, round 9's admitted-size linear-views case ran ten 8-MiB projections in one case +and passed its 15 s case deadline on the runner (3.7 s plain and 7.2 s under local coverage). `2a1f7359` +makes it one case per shape with the same inputs, assertions and deadline (1.1–2.8 s each under local +coverage; 221 core projection cases), and the local enforced coverage gate passes at llm 97.79 % lines +and 95.69 % branches, mcp 96.56 % and 91.29 %, core (measured) 96.71 % and 91.06 %. The author now also +runs the coverage gate before a push. CI at `2a1f7359` then passed every job but the filesystem causal +controls, whose export refused: each phase retains a manifest of the whole source graph (203,421 bytes +once PR95 grew it), and the 330 manifests of one export passed the unchanged 64-MiB bound by 20,066 +bytes. `4c3cafe9` runs that family as two exporting shards, as PR95's round 7 did for the secrets family +([filesystem controls README](../../../tools/fs-policy-controls/README.md)), and CI at `4c3cafe9` passed +every job, both shards of each control family, the coverage floor and the four installed-CLI consumers +included. + +### Wave 3 Step 9a.1 implementation checkpoint — 2026-10-22 + +**Scope and source.** Step 9a.1 (the 9a.1 row of the increment table; #46, ADR-0111) implements the +candidate build and finalizer on `development`, on top of `4c3cafe9`, the PR95 closure and its two CI +corrections. It changes +`tools/cli-package` (the owned candidate tree `candidate.mjs`; the producer `produce.mjs` and its entry +point `pack.mjs`; the prepack orchestrator `prepack.mjs`; the finalizer `finalize.mjs`; the seal; the +archive and smoke fold-ins; the interim publication guard `release-guard.mjs` with its rule +`release-guard-core.mjs`), `tools/bundle-closure` (map-aware closure and a shared emitted inventory), +`apps/cli` (the prepack script and the candidate-mode tsup selection) and `.github/workflows/release.yml` +(the guard before npm publication), with the +[packaging contract](../../reference/cli/packaging.md#candidate-build-and-finalizer-implementation-boundary--step-9a1), +the release runbook, an ADR-0051 amendment and the tooling README. It adds no runtime dependency, and +the receipt stays `compiled_package_test`; the contract section is the canonical description of the +mechanism. + +**Before the commit.** An internal read-only pre-review of the prepared change (five lenses, adversarial +verification of every finding and a completeness critic; the detailed record is local) found seventeen +issues, none high once verified. The publication guard refused only when it recognized itself as the +entry point, so a symlinked or otherwise non-canonical path would have let a tag push reach +`npm publish`; the producer's composition had no test; the candidate's file list ignored the developer's +global Git excludes, so a globally ignored private file entered the candidate; and smaller gaps in tests, +modes and wording. All are corrected: the guard's entry point exits 1 before it evaluates its rule, the +producer is a tested module, the candidate reads the developer's global excludes file and nothing else +of the global configuration, the candidate copy's modes are exact under any umask, and each gap has a +killing test. A mutation run of twenty mutants over the new code kills all of them. The full local CI +then found one more defect in the change itself: `tools/bundle-closure/closure.test.mjs` built its +staged fixture with a top-level `await` after most of its tests were registered, so a slow build let the +runner finish those tests and run the file's `after` hook, which removed the scratch directory a later +test needed (intermittent under load; reproduced deterministically with a delayed fixture). Both +fixtures are now built before the first test is registered, as the file's first fixture already was. + +**Author gates.** `lint:tools` and `typecheck:tools` pass, the 259 cases of the changed tooling tests +pass, and the full `pnpm run ci` on top of `4c3cafe9` passes, both exporting secrets shards included +(65 controls each, 50.39 MiB and 50.28 MiB), with the producer running for real in `smoke:cli-package`. +Under `CI=true GITHUB_ACTIONS=true`, 709 of the 710 acceptance tooling tests pass; the other one, +unchanged by this step, installs through the corepack pnpm shim in an empty home, which needs the +registry, and corepack hung for 45 s on this host while the same tree had passed it in the full run an +hour earlier. + +**Remaining gates.** The increment is not independently accepted. Two NEW consecutive clean complete +cumulative rounds after the last material fix and a separate security sitting remain (the Acceptance +row of the 2026-10-19 rulings). Then `development` is pushed and its CI observed, and the PR #95 +tooling nits in +[deferred-tasks.md](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18) +close. diff --git a/docs/runbooks/release-a-surface.md b/docs/runbooks/release-a-surface.md index 0a402263..8b8da656 100644 --- a/docs/runbooks/release-a-surface.md +++ b/docs/runbooks/release-a-surface.md @@ -47,6 +47,14 @@ and the bundle boundary are [ADR-0051](../decisions/0051-cli-distribution-thin-b **This is the first release flow; desktop and VS Code inherit its shape (pack → cross-OS smoke → publish).** +> **Publication is refused — interim, 2026-10-22 (Wave 3 Step 9a.1).** The `publish` job first runs +> a fail-closed guard that refuses every artifact: only a verified release candidate (exact tarball, +> original ciphertext and release binding receipt) may publish, and none can exist until Wave 3 Step +> 9d's custody graph and Wave 4b's production recipient do, so a tag push stops after the cross-OS smoke and nothing — not +> even a v0.1.x patch — reaches npm (the [Interim release guard ruling](../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) of 2026-10-19). +> The steps below describe the pre-custody flow; Step 9d replaces them with the +> [custody graph](../reference/cli/packaging.md#archive-retry-and-publication). + ### What the build produces - `apps/cli/dist/index.js` — the single ESM bin (shebang, minified, **no** source map: a map would ship the diff --git a/package.json b/package.json index efa79ef4..569e6b9c 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "smoke:budget-replay": "node tools/budget-replay-compat/check.mjs", "check:wire-mutations": "node tools/wire-conformance/check.mjs", "check:usage-mutations": "node tools/usage-coverage-controls/check.mjs", - "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/consumer-lifetime.test.mjs", + "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", "check:fs-mutations": "node tools/fs-policy-controls/check.mjs", "check:secrets-mutations": "node tools/secrets-controls/check.mjs", "smoke:cli-bootstrap": "node tools/cli-bootstrap/check.mjs", diff --git a/tools/bundle-closure/check.mjs b/tools/bundle-closure/check.mjs index 2aa9bb81..85887c96 100644 --- a/tools/bundle-closure/check.mjs +++ b/tools/bundle-closure/check.mjs @@ -1,45 +1,24 @@ -/** Thin filesystem/exit adapter for the ADR-0111 emitted ESM graph guard. */ -import { lstatSync, readFileSync, readdirSync } from 'node:fs'; -import { join, relative, sep } from 'node:path'; +/** Thin filesystem/exit adapter for the ADR-0111 emitted ESM graph guard of an ordinary build. */ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { BundleClosureError, checkBundleClosure } from './closure.mjs'; +import { MODE_EVIDENCE, emittedInventory } from './inventory.mjs'; -const modeEvidence = process.platform === 'win32' ? 'not-observable-windows' : 'posix'; const packageDirectory = fileURLToPath(new URL('../../apps/cli/', import.meta.url)); -function inventory(directory, files = {}) { - if (!lstatSync(directory).isDirectory()) { - throw new BundleClosureError('OUTPUT_KIND', 'dist inventory must use ordinary directories'); - } - for (const entry of readdirSync(directory, { withFileTypes: true })) { - const path = join(directory, entry.name); - if (entry.isDirectory()) inventory(path, files); - else if (/\.(?:js|mjs|cjs)$/iu.test(entry.name)) { - const key = relative(packageDirectory, path).split(sep).join('/'); - const stat = lstatSync(path); - files[key] = { - contents: entry.isFile() ? readFileSync(path, 'utf8') : '', - ...(modeEvidence === 'posix' ? { mode: stat.mode & 0o7777 } : {}), - kind: entry.isFile() ? 'file' : 'other', - }; - } else if (entry.isSymbolicLink()) { - // Never follow a directory symlink that could hide runtime chunks. - throw new BundleClosureError('OUTPUT_KIND', 'dist contains a symbolic link'); - } - } - return files; -} - try { + // An ordinary build is map-free: its metafile is the only unpublished file it may leave in dist. const report = checkBundleClosure({ metafile: JSON.parse(readFileSync(join(packageDirectory, 'dist/metafile-esm.json'), 'utf8')), manifest: JSON.parse(readFileSync(join(packageDirectory, 'package.json'), 'utf8')), - files: inventory(join(packageDirectory, 'dist')), - modeEvidence, + files: emittedInventory(packageDirectory, { privateFiles: ['metafile-esm.json'] }), + modeEvidence: MODE_EVIDENCE, + maps: 'none', }); console.log( `✓ CLI finite ESM closure: ${report.outputs.length} output(s), ${report.inputs.length} input(s), ` + - `${report.dependencies.length} declared external dependencies; engine inlined; ` + + `${report.dependencies.length} declared external dependencies; engine inlined; map-free; ` + (report.modeEvidence === 'posix' ? 'POSIX modes verified.' : 'POSIX modes not observable on Windows (graph evidence only).'), diff --git a/tools/bundle-closure/closure.mjs b/tools/bundle-closure/closure.mjs index 95e1ccf6..f726f639 100644 --- a/tools/bundle-closure/closure.mjs +++ b/tools/bundle-closure/closure.mjs @@ -1,11 +1,13 @@ /** * Finite emitted ESM graph guard (ADR-0111). No filesystem reads or process state: * the CLI supplies an inventory, and controls can remove/change any independent - * witness. This checks emitted files, not a tarball or Step 9 map custody. + * witness. This checks emitted files, not a tarball. A finalized candidate's maps are + * checked here only for pairing; the candidate finalizer owns their content and custody. */ import { Buffer } from 'node:buffer'; import { isBuiltin } from 'node:module'; import { posix } from 'node:path'; +import { isDeepStrictEqual } from 'node:util'; import ts from 'typescript'; export class BundleClosureError extends Error { @@ -203,11 +205,55 @@ function actualImports(contents, path) { return edges; } +/** + * Separate external maps from runtime outputs. An ordinary build is map-free. A finalized + * candidate pairs every runtime output with exactly one map, staged outside the package, whose + * metadata entry carries no graph information of its own. + */ +function runtimeOutputs(outputs, directory, maps) { + const runtime = {}; + const staged = new Set(); + for (const [path, output] of Object.entries(outputs)) { + if (!path.endsWith('.map')) { + runtime[path] = output; + continue; + } + if (maps !== 'staged') refuse('MAP_OUTPUT', `ordinary build emitted a source map: ${path}`); + outputPath(path.slice(0, -'.map'.length), directory); + const entry = record(output, `map output ${path}`); + bytes(entry.bytes, `map output ${path}.bytes`); + if ( + !isDeepStrictEqual(Object.keys(entry).sort(), ['bytes', 'exports', 'imports', 'inputs']) || + Object.keys(record(entry.inputs, `map output ${path}.inputs`)).length > 0 || + strings(entry.exports, `map output ${path}.exports`).length > 0 || + imports(entry.imports, `map output ${path}`).length > 0 + ) { + refuse('MAP_OUTPUT', `staged map metadata carries graph information: ${path}`); + } + staged.add(path); + } + if (maps === 'staged') { + for (const path of Object.keys(runtime)) { + if (!staged.has(`${path}.map`)) { + refuse('MAP_PAIRING', `runtime output has no staged map: ${path}`); + } + } + for (const path of staged) { + if (!Object.hasOwn(runtime, path.slice(0, -'.map'.length))) { + refuse('MAP_PAIRING', `staged map has no runtime output: ${path}`); + } + } + } + return runtime; +} + /** * @param {{metafile: unknown, manifest: unknown, * files: Record, * outputDirectory?: string, entryPoint?: string, - * modeEvidence?: "posix" | "not-observable-windows"}} options + * modeEvidence?: "posix" | "not-observable-windows", + * maps?: "none" | "staged"}} options `staged` admits a finalized candidate's paired map + * metadata; the map files themselves must already be outside the inventoried package. * @returns {{outputs: string[], dependencies: string[], inputs: string[], modeEvidence: string}} * @throws {BundleClosureError} when an emitted graph invariant fails. */ @@ -218,6 +264,7 @@ export function checkBundleClosure({ outputDirectory = 'dist', entryPoint = 'src/index.ts', modeEvidence = 'posix', + maps = 'none', }) { if (!/^[a-zA-Z0-9_-]+$/u.test(outputDirectory)) { refuse('OUTPUT_PATH', 'outputDirectory must be one plain directory name'); @@ -225,10 +272,13 @@ export function checkBundleClosure({ if (!['posix', 'not-observable-windows'].includes(modeEvidence)) { refuse('MODE_EVIDENCE', 'mode evidence must name its observable platform scope'); } + if (!['none', 'staged'].includes(maps)) { + refuse('MAP_MODE', 'map mode must be none or staged'); + } sourcePath(entryPoint, 'entryPoint'); const meta = record(metafile, 'metafile'); const inputs = record(meta.inputs, 'metafile.inputs'); - const outputs = record(meta.outputs, 'metafile.outputs'); + const outputs = runtimeOutputs(record(meta.outputs, 'metafile.outputs'), outputDirectory, maps); const dependencies = record(record(manifest, 'manifest').dependencies, 'manifest.dependencies'); record(files, 'files'); const declared = new Set(Object.keys(dependencies)); @@ -310,6 +360,10 @@ export function checkBundleClosure({ } else if (file.mode !== undefined) { refuse('MODE_EVIDENCE', 'Windows graph evidence cannot assert unobservable POSIX modes'); } + // Neither an ordinary build nor a finalized candidate may link a map from runtime code. + if (/sourceMappingURL\s*=/u.test(file.contents)) { + refuse('MAP_LINK', `runtime output links a source map: ${path}`); + } if (Buffer.byteLength(file.contents, 'utf8') !== output.bytes) { refuse('OUTPUT_BYTES', `output byte inventory disagrees: ${path}`); } diff --git a/tools/bundle-closure/closure.test.mjs b/tools/bundle-closure/closure.test.mjs index 404bb0a4..6af1b7b0 100644 --- a/tools/bundle-closure/closure.test.mjs +++ b/tools/bundle-closure/closure.test.mjs @@ -17,6 +17,7 @@ import { dirname, join, relative, sep } from 'node:path'; import { after, test } from 'node:test'; import { fileURLToPath } from 'node:url'; import { BundleClosureError, checkBundleClosure } from './closure.mjs'; +import { emittedInventory } from './inventory.mjs'; // Use the exact esbuild installed behind the existing CLI tsup tool, not a new // dependency, a copied metadata generator, or the product build. Everything is @@ -77,6 +78,47 @@ async function fixture(splitting = true) { }; } const original = await fixture(); + +// Every fixture is built before the first test is registered: a top-level await after a registration +// lets the runner finish the earlier tests and run this file's `after` hook, which removes `scratch`. +// A finalized candidate (Wave 3 Step 9a.1): esbuild emitted external maps that the finalizer moved +// out of the package; their metafile entries remain and must pair exactly with runtime outputs. +async function stagedFixture() { + const built = await build({ + absWorkingDir: scratch, + entryPoints: { index: 'src/index.ts' }, + outdir: 'dist', + format: 'esm', + platform: 'node', + target: 'node22', + bundle: true, + splitting: true, + minify: true, + metafile: true, + write: false, + sourcemap: 'external', + sourcesContent: false, + external: ['commander', '@modelcontextprotocol/sdk', '@modelcontextprotocol/sdk/*'], + }); + const files = {}; + for (const output of built.outputFiles) { + const path = relative(scratch, output.path).split(sep).join('/'); + if (path.endsWith('.map')) continue; + files[path] = { + contents: output.text, + mode: path === 'dist/index.js' ? 0o755 : 0o644, + kind: 'file', + }; + } + return { + metafile: built.metafile, + manifest: { dependencies: { commander: '1', '@modelcontextprotocol/sdk': '1' } }, + files, + maps: 'staged', + }; +} +const stagedOriginal = await stagedFixture(); +const staged = () => JSON.parse(JSON.stringify(stagedOriginal)); function candidate() { return JSON.parse(JSON.stringify(original)); } @@ -442,7 +484,7 @@ test('thin CLI inventories actual files from its package root and refuses missin join(root, 'node_modules'), process.platform === 'win32' ? 'junction' : 'dir', ); - for (const name of ['check.mjs', 'closure.mjs']) { + for (const name of ['check.mjs', 'closure.mjs', 'inventory.mjs']) { copyFileSync(fileURLToPath(new URL(name, import.meta.url)), join(tooling, name)); } writeFileSync(join(pkg, 'package.json'), JSON.stringify(original.manifest)); @@ -461,6 +503,14 @@ test('thin CLI inventories actual files from its package root and refuses missin return result; } assert.equal(run().status, 0); + // An ordinary build is map-free: a planted map or unreviewed file in dist refuses. + writeFileSync(join(pkg, 'dist/index.js.map'), '{}'); + assert.match(run().stderr, /MAP_OUTPUT/u); + rmSync(join(pkg, 'dist/index.js.map')); + writeFileSync(join(pkg, 'dist/notes.txt'), 'x'); + assert.match(run().stderr, /EXTRA_OUTPUT/u); + rmSync(join(pkg, 'dist/notes.txt')); + assert.equal(run().status, 0); writeFileSync(join(pkg, 'dist/extra.js'), 'export {};'); assert.match(run().stderr, /EXTRA_OUTPUT/u); rmSync(join(pkg, 'dist/extra.js')); @@ -490,3 +540,116 @@ test('POSIX modes require actual evidence; Windows graph evidence is explicitly missing.modeEvidence = 'unverified'; refuses(missing, 'MODE_EVIDENCE'); }); + +test('a finalized candidate passes with paired staged maps; an ordinary build may not carry maps', () => { + const value = staged(); + const maps = Object.keys(value.metafile.outputs).filter((path) => path.endsWith('.map')); + assert.ok(maps.length >= 4); + assert.equal(checkBundleClosure(value).outputs.length, maps.length); + value.maps = 'none'; + refuses(value, 'MAP_OUTPUT'); + const implicit = staged(); + delete implicit.maps; + refuses(implicit, 'MAP_OUTPUT'); +}); + +for (const [label, mutate, code] of [ + [ + 'an unpaired runtime output', + (value) => delete value.metafile.outputs['dist/index.js.map'], + 'MAP_PAIRING', + ], + [ + 'an orphan map', + (value) => + (value.metafile.outputs['dist/orphan.js.map'] = { + imports: [], + exports: [], + inputs: {}, + bytes: 1, + }), + 'MAP_PAIRING', + ], + [ + 'map metadata with inputs', + (value) => + (value.metafile.outputs['dist/index.js.map'].inputs = { + 'src/index.ts': { bytesInOutput: 1 }, + }), + 'MAP_OUTPUT', + ], + [ + 'map metadata with imports', + (value) => + value.metafile.outputs['dist/index.js.map'].imports.push({ + path: 'commander', + kind: 'import-statement', + external: true, + }), + 'MAP_OUTPUT', + ], + [ + 'map metadata with an entry point', + (value) => (value.metafile.outputs['dist/index.js.map'].entryPoint = 'src/index.ts'), + 'MAP_OUTPUT', + ], + [ + 'a nested map', + (value) => + (value.metafile.outputs['dist/nested/index.js.map'] = { + imports: [], + exports: [], + inputs: {}, + bytes: 1, + }), + 'OUTPUT_PATH', + ], + ['an unknown map mode', (value) => (value.maps = 'external'), 'MAP_MODE'], +]) { + test(`staged closure refuses ${label}`, () => { + const value = staged(); + mutate(value); + refuses(value, code); + }); +} + +test('runtime code may never link a map, in an ordinary build or a finalized candidate', () => { + for (const value of [candidate(), staged()]) { + setContents( + value, + 'dist/index.js', + `${value.files['dist/index.js'].contents}//# sourceMappingURL=index.js.map`, + ); + refuses(value, 'MAP_LINK'); + } +}); + +test('the emitted inventory refuses maps and unreviewed files and admits only named private files', () => { + const pkg = mkdtempSync(join(scratch, 'inventory-')); + mkdirSync(join(pkg, 'dist')); + writeFileSync(join(pkg, 'dist/index.js'), '#!/usr/bin/env node\n'); + writeFileSync(join(pkg, 'dist/metafile-esm.json'), '{}'); + assert.deepEqual(Object.keys(emittedInventory(pkg, { privateFiles: ['metafile-esm.json'] })), [ + 'dist/index.js', + ]); + // An unnamed build file is not private: a finalized candidate's dist holds no metafile at all. + assert.throws( + () => emittedInventory(pkg), + (error) => error instanceof BundleClosureError && error.code === 'EXTRA_OUTPUT', + ); + for (const [path, code] of [ + ['dist/index.js.map', 'MAP_OUTPUT'], + ['dist/nested/chunk.js.map', 'MAP_OUTPUT'], + ['dist/notes.txt', 'EXTRA_OUTPUT'], + ['dist/nested/metafile-esm.json', 'EXTRA_OUTPUT'], + ]) { + mkdirSync(dirname(join(pkg, path)), { recursive: true }); + writeFileSync(join(pkg, path), 'x'); + assert.throws( + () => emittedInventory(pkg, { privateFiles: ['metafile-esm.json'] }), + (error) => error instanceof BundleClosureError && error.code === code, + path, + ); + rmSync(join(pkg, path)); + } +}); diff --git a/tools/bundle-closure/inventory.mjs b/tools/bundle-closure/inventory.mjs new file mode 100644 index 00000000..2bfc1ade --- /dev/null +++ b/tools/bundle-closure/inventory.mjs @@ -0,0 +1,47 @@ +/** Filesystem inventory for the ADR-0111 emitted graph guard. Never follows a link. */ +import { lstatSync, readFileSync, readdirSync } from 'node:fs'; +import { join, relative, sep } from 'node:path'; +import { BundleClosureError } from './closure.mjs'; + +/** POSIX modes are actual evidence only where the platform exposes them. */ +export const MODE_EVIDENCE = process.platform === 'win32' ? 'not-observable-windows' : 'posix'; + +/** + * Inventory every emitted JS file below `/dist`. A map, an unknown file kind or a + * private build file other than the named ones refuses: an ordinary build is map-free, and a + * finalized candidate has already moved its maps and build metadata out of the package. + * @param {string} packageDirectory + * @param {{privateFiles?: readonly string[]}} [options] dist-root files that may remain unpublished + * @returns {Record} + */ +export function emittedInventory(packageDirectory, { privateFiles = [] } = {}) { + const files = {}; + const root = join(packageDirectory, 'dist'); + function visit(directory) { + if (!lstatSync(directory).isDirectory()) { + throw new BundleClosureError('OUTPUT_KIND', 'dist inventory must use ordinary directories'); + } + for (const entry of readdirSync(directory, { withFileTypes: true })) { + const path = join(directory, entry.name); + const key = relative(packageDirectory, path).split(sep).join('/'); + if (entry.isDirectory()) visit(path); + else if (entry.isSymbolicLink()) { + // Never follow a link that could hide runtime chunks. + throw new BundleClosureError('OUTPUT_KIND', 'dist contains a symbolic link'); + } else if (/\.(?:js|mjs|cjs)$/iu.test(entry.name)) { + const stat = lstatSync(path); + files[key] = { + contents: entry.isFile() ? readFileSync(path, 'utf8') : '', + ...(MODE_EVIDENCE === 'posix' ? { mode: stat.mode & 0o7777 } : {}), + kind: entry.isFile() ? 'file' : 'other', + }; + } else if (/\.map$/iu.test(entry.name)) { + throw new BundleClosureError('MAP_OUTPUT', `dist contains a source map: ${key}`); + } else if (!(directory === root && privateFiles.includes(entry.name))) { + throw new BundleClosureError('EXTRA_OUTPUT', `dist contains an unreviewed file: ${key}`); + } + } + } + visit(root); + return files; +} diff --git a/tools/ci/require-packed-cli.test.mjs b/tools/ci/require-packed-cli.test.mjs index 94272154..6f443ebb 100644 --- a/tools/ci/require-packed-cli.test.mjs +++ b/tools/ci/require-packed-cli.test.mjs @@ -268,8 +268,36 @@ test('release keeps main ancestry, version, real-push publication and npm proven ); assert.deepEqual( jobs.publish.steps.filter((step) => step.run).map((step) => step.run), - ['npm publish ./artifact/*.tgz --provenance --access public'], + [ + 'node tools/cli-package/release-guard.mjs ./artifact', + 'npm publish ./artifact/*.tgz --provenance --access public', + ], + ); +}); + +test('release publication is refused fail-closed until a verified release candidate exists', () => { + const steps = readWorkflow('release').jobs.publish.steps; + const checkout = steps.find((step) => step.uses === actions.checkout); + assert.deepEqual(checkout.with, { + 'persist-credentials': false, + 'sparse-checkout': + '.nvmrc\ntools/cli-package/release-guard.mjs\ntools/cli-package/release-guard-core.mjs\n', + 'sparse-checkout-cone-mode': false, + }); + const guard = steps.find( + (step) => step.name === 'Refuse anything but a verified release candidate', + ); + const publish = steps.find((step) => step.name === 'Publish'); + assert.equal(guard.run, 'node tools/cli-package/release-guard.mjs ./artifact'); + // An unconditional, non-advisory guard runs after the download and before any publish. + assert.equal(guard.if, undefined); + assert.equal(guard['continue-on-error'], undefined); + assert.equal(guard.env, undefined); + assert.ok( + steps.indexOf(steps.find((step) => step.uses === actions.download)) < steps.indexOf(guard), ); + assert.equal(steps.indexOf(guard) + 1, steps.indexOf(publish)); + assert.equal(publish.if, undefined); }); for (const [label, alter] of [ diff --git a/tools/cli-package/README.md b/tools/cli-package/README.md index b6520200..abb5a760 100644 --- a/tools/cli-package/README.md +++ b/tools/cli-package/README.md @@ -6,7 +6,8 @@ The canonical package and release contract lives in [packaging.md](../../docs/reference/cli/packaging.md). Step 8b adds required creation-owned native process controls to the same installed artifact cohort. Step 8d adds installed global fatal/background/output-hook -controls; complete Step 8d acceptance remains separate. +controls; complete Step 8d acceptance remains separate. Step 9a.1 makes the producer +build an owned map-bearing candidate and finalize it before packing. ```sh pnpm package:cli:pack /absolute/empty/artifact-directory @@ -14,9 +15,17 @@ pnpm package:cli:consume /absolute/artifact-directory /absolute/fresh/report-dir pnpm smoke:cli-package ``` -The producer emits the five inlined dependency packages in order with the installed -compiler and no build cache. Prepack performs the sole final CLI build. The runner -observes one actual prepack build-start record, checks the emitted graph, and +The producer (`produce.mjs`, run by `pack.mjs`) never builds in the developer's workspace. +It copies the files Git would ship, by the repository's ignore rules and the developer's +global excludes file, into a fresh owner-only candidate tree, refusing if the original +checkout's commit or status changes during the copy, and installs the locked closure there offline, frozen and +without lifecycle scripts from the existing store. In that tree it emits the five +inlined dependency packages in order with the installed compiler, upstream maps and no +build cache. Prepack (`prepack.mjs`) performs the sole final CLI build in candidate +mode and runs the finalizer (`finalize.mjs`), which validates every upstream and final +map, removes only the exact map-link trailers and stages maps, metafile and its hash +record in a private directory outside the artifact. The runner observes one actual +prepack build-start record and one finalization, checks the emitted graph, and independently inspects the produced gzip/USTAR bytes. The inspector never extracts the archive. Every JS and migration byte must match the finalized build and canonical migrations; package metadata must match pnpm's publication transform. @@ -28,9 +37,13 @@ controls exclude links, traversal, duplicates, extra files, maps and private bui inventories. This deliberately supports the pinned packer's dialect, not general tar. The resulting public `package-test-receipt.json` records `compiled_package_test`. -It does **not** certify original-TypeScript maps, symbolication, encrypted custody -or a production release candidate. Those are Step 9 obligations. No private map, -metafile or per-source inventory enters the public artifact. +The candidate's maps are built, validated and then discarded with the rest of the +private staging; the receipt does **not** certify symbolication, a map manifest, +encrypted custody or a production release candidate (Steps 9a.2–9d). No private map, +metafile or per-source inventory enters the public artifact. The release workflow's +`release-guard.mjs` refuses every artifact until verified release custody exists: it +exits 1 before it evaluates its rule (`release-guard-core.mjs`), so no way of invoking it +can skip the refusal. The consumer verifies the chosen bytes before installation, installs a copy outside the workspace, and compares every installed public file with the archive receipt. diff --git a/tools/cli-package/archive.mjs b/tools/cli-package/archive.mjs index 6b2afa0c..9cf20999 100644 --- a/tools/cli-package/archive.mjs +++ b/tools/cli-package/archive.mjs @@ -123,7 +123,6 @@ function readHeader(header) { [265, 329], [345, 512], ]) { - if (start === undefined || end === undefined) refuse('TAR_FORMAT'); if (header.subarray(start, end).some((byte) => byte !== 0)) refuse('TAR_METADATA'); } for (const [start, end] of [ @@ -132,7 +131,6 @@ function readHeader(header) { [329, 337], [337, 345], ]) { - if (start === undefined || end === undefined) refuse('TAR_FORMAT'); if (octal(header.subarray(start, end)) !== 0) refuse('TAR_METADATA'); } octal(header.subarray(136, 148)); // Validate mtime syntax without requiring a clock value. diff --git a/tools/cli-package/candidate.mjs b/tools/cli-package/candidate.mjs new file mode 100644 index 00000000..864b57cf --- /dev/null +++ b/tools/cli-package/candidate.mjs @@ -0,0 +1,338 @@ +/** + * The owned isolated candidate tree (ADR-0111, Wave 3 Step 9a.1). A release-map build never runs in + * the developer's workspace: the producer copies the exact listed source files of the original + * checkout into a fresh private tree, installs the locked dependency closure there from the existing + * content-addressed store without lifecycle scripts or network, and verifies that every workspace + * link resolves inside that tree. No original workspace `dist` can stand in for freshly mapped + * output, and no build cache is read or written. + */ +import assert from 'node:assert/strict'; +import { + chmodSync, + existsSync, + lstatSync, + mkdirSync, + readdirSync, + realpathSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { dirname, isAbsolute, join, relative, resolve } from 'node:path'; +import { ENGINE_PACKAGES } from './finalize.mjs'; +import { readOrdinaryFile } from './files.mjs'; +import { isolatedEnvironment, runCommand, runPnpm } from './process.mjs'; + +export const CANDIDATE_LIMITS = Object.freeze({ + files: 20_000, + fileBytes: 64 * 1024 * 1024, + totalBytes: 512 * 1024 * 1024, +}); + +/** Only the variables pnpm uses to locate its store; never credentials, preloads or registries. */ +const STORE_DISCOVERY_VARIABLES = Object.freeze([ + 'HOME', + 'USERPROFILE', + 'XDG_DATA_HOME', + 'XDG_CONFIG_HOME', + 'LOCALAPPDATA', + 'APPDATA', + 'PNPM_HOME', +]); + +/** @param {string} parent @param {string} child */ +function within(parent, child) { + const path = relative(parent, child); + return path === '' || (!path.startsWith('..') && !isAbsolute(path)); +} + +/** @param {string} output NUL-separated Git path list */ +function gitPaths(output) { + return output.split('\0').filter((path) => path !== ''); +} + +/** Names the developer's global Git excludes file for a producer whose own environment is isolated. */ +export const GIT_EXCLUDES_ENV = 'RELAVIUM_GIT_EXCLUDES_FILE'; + +/** Only the variables Git uses to locate its global configuration; never credentials or hooks. */ +const GIT_CONFIG_VARIABLES = Object.freeze([ + 'HOME', + 'USERPROFILE', + 'XDG_CONFIG_HOME', + 'GIT_CONFIG_GLOBAL', + 'GIT_CONFIG_SYSTEM', + 'GIT_CONFIG_NOSYSTEM', + 'SystemRoot', + 'WINDIR', +]); + +/** + * The developer's global Git excludes file, so that the candidate's isolated Git ignores what the + * developer's own `git status` ignores: the explicit `RELAVIUM_GIT_EXCLUDES_FILE` when a caller + * already isolated this process's environment, otherwise `core.excludesFile` as the developer's Git + * reads it, else Git's default `$XDG_CONFIG_HOME/git/ignore` or `$HOME/.config/git/ignore`; undefined + * when no such file exists. Only that location is read from the developer's configuration: every + * candidate Git command keeps the isolated environment, so no global hook, fsmonitor or other setting + * applies. + * @param {{repository: string, environment: Record, run?: typeof runCommand}} options + * @returns {Promise} + */ +export async function globalExcludesFile({ repository, environment, run = runCommand }) { + const explicit = environment[GIT_EXCLUDES_ENV]; + if (explicit !== undefined) { + assert.ok(isAbsolute(explicit) && existsSync(explicit), 'PACKED_CLI_GIT_EXCLUDES'); + return explicit; + } + const env = { PATH: environment.PATH ?? '' }; + for (const key of GIT_CONFIG_VARIABLES) { + if (environment[key] !== undefined) env[key] = environment[key]; + } + let configured = ''; + try { + configured = ( + await run('git', ['config', '--path', '--get', 'core.excludesFile'], { cwd: repository, env }) + ).stdout.trim(); + } catch (error) { + // Git exits 1 when the setting is absent; anything else is a refusal. + if (error?.result?.code !== 1) throw error; + } + const xdg = environment.XDG_CONFIG_HOME; + const home = environment.HOME ?? environment.USERPROFILE; + let path; + if (configured !== '') path = resolve(repository, configured); + else if (xdg !== undefined && xdg !== '') path = join(xdg, 'git', 'ignore'); + else if (home !== undefined && home !== '') path = join(home, '.config', 'git', 'ignore'); + return path !== undefined && existsSync(path) && statSync(path).isFile() ? path : undefined; +} + +/** @param {string | undefined} excludes @returns {string[]} */ +function excludesArguments(excludes) { + return excludes === undefined ? [] : ['-c', `core.excludesFile=${excludes}`]; +} + +/** + * The source identity of the original checkout: its exact commit and its full porcelain status, so + * a candidate copied from a modified tree records `dirty` and a status change during the copy + * refuses. Untracked files are judged with the developer's global excludes (`globalExcludesFile`). + * @param {string} repository @param {typeof runCommand} [run] @param {string} [excludes] + */ +export async function sourceIdentity(repository, run = runCommand, excludes = undefined) { + const options = { cwd: repository, env: isolatedEnvironment(repository) }; + const source = ( + await run('git', ['--no-replace-objects', 'rev-parse', 'HEAD'], options) + ).stdout.trim(); + assert.match(source, /^[0-9a-f]{40}$/u, 'PACKED_CLI_SOURCE_IDENTITY'); + const status = ( + await run( + 'git', + [...excludesArguments(excludes), 'status', '--porcelain=v1', '-z', '--untracked-files=all'], + options, + ) + ).stdout; + return { source, status, dirty: status !== '' }; +} + +/** + * Every file Git would ship from the working tree: tracked plus untracked-but-not-ignored, minus + * deletions, by the repository's ignore rules and the developer's global excludes + * (`globalExcludesFile`). Ignored output, caches, dependencies and private notes never enter a + * candidate. + * @param {string} repository @param {typeof runCommand} [run] @param {string} [excludes] + */ +export async function candidateFiles(repository, run = runCommand, excludes = undefined) { + const options = { cwd: repository, env: isolatedEnvironment(repository) }; + const listed = gitPaths( + ( + await run( + 'git', + [ + ...excludesArguments(excludes), + 'ls-files', + '-z', + '--cached', + '--others', + '--exclude-standard', + ], + options, + ) + ).stdout, + ); + const deleted = new Set( + gitPaths((await run('git', ['ls-files', '-z', '--deleted'], options)).stdout), + ); + const files = [...new Set(listed)].filter((path) => !deleted.has(path)).sort(); + assert.ok( + files.length > 0 && files.length <= CANDIDATE_LIMITS.files, + 'PACKED_CLI_CANDIDATE_FILES', + ); + return files; +} + +/** + * Copy each listed ordinary file byte-for-byte into the empty `destination`. Links, special files, + * escaping or noncanonical paths and oversized trees refuse; only the executable bit is kept, as an + * exact 0755 or 0644 mode whatever the process umask. + * @param {{repository: string, destination: string, files: readonly string[]}} options + */ +export function copyCandidateTree({ repository, destination, files }) { + assert.ok( + isAbsolute(destination) && readdirSync(destination).length === 0, + 'PACKED_CLI_CANDIDATE_ROOT', + ); + let total = 0; + for (const path of files) { + assert.ok( + typeof path === 'string' && + path.length > 0 && + !path.includes('\\') && + path.split('/').every((part) => part !== '' && part !== '.' && part !== '..'), + 'PACKED_CLI_CANDIDATE_PATH', + ); + const source = join(repository, ...path.split('/')); + const executable = (lstatSync(source).mode & 0o111) !== 0; + const bytes = readOrdinaryFile(source, CANDIDATE_LIMITS.fileBytes); + total += bytes.length; + assert.ok(total <= CANDIDATE_LIMITS.totalBytes, 'PACKED_CLI_CANDIDATE_BYTES'); + const target = join(destination, ...path.split('/')); + mkdirSync(dirname(target), { recursive: true, mode: 0o700 }); + const mode = executable ? 0o755 : 0o644; + writeFileSync(target, bytes, { flag: 'wx', mode }); + // The creation mode passes through the umask; the candidate keeps the exact mode. + if (process.platform !== 'win32') chmodSync(target, mode); + } + return { files: files.length, bytes: total }; +} + +/** An explicit store directory, set by a caller that discovered it in its own environment. */ +export const PACKAGE_STORE_ENV = 'RELAVIUM_PACKAGE_STORE_DIR'; + +/** + * The developer's existing content-addressed dependency store: the explicit + * `RELAVIUM_PACKAGE_STORE_DIR` when a caller already isolated this process's environment, + * otherwise discovered by the pinned pnpm with only the store-location variables of `environment`. + * Everything else — temporary files, caches, corepack state — goes to the private `home`, never + * into the original checkout. + * @param {{repository: string, environment: Record, home: string, + * pnpm?: typeof runPnpm}} options + */ +export async function dependencyStore({ repository, environment, home, pnpm = runPnpm }) { + const explicit = environment[PACKAGE_STORE_ENV]; + if (explicit !== undefined) { + assert.ok(isAbsolute(explicit) && existsSync(explicit), 'PACKED_CLI_DEPENDENCY_STORE'); + return explicit; + } + assert.ok(isAbsolute(home) && !within(repository, home), 'PACKED_CLI_PRIVATE_HOME'); + const env = isolatedEnvironment(home); + for (const key of STORE_DISCOVERY_VARIABLES) { + if (environment[key] === undefined) delete env[key]; + else env[key] = environment[key]; + } + const store = (await pnpm(['store', 'path'], { cwd: repository, env })).stdout.trim(); + assert.ok(isAbsolute(store) && existsSync(store), 'PACKED_CLI_DEPENDENCY_STORE'); + return store; +} + +/** + * Install the locked closure into the candidate from the store: offline, frozen and without any + * lifecycle script, because building or packing the CLI never loads a native module. + * @param {{candidate: string, store: string, env: Record, pnpm: typeof runPnpm}} options + */ +export async function installCandidateDependencies({ candidate, store, env, pnpm }) { + await pnpm( + ['install', '--offline', '--frozen-lockfile', '--ignore-scripts', '--store-dir', store], + { cwd: candidate, env }, + ); +} + +/** + * Every workspace link must resolve inside the candidate, never to the original checkout whose + * `dist` is map-free and possibly stale. + * @param {string} candidate + */ +export function verifyCandidateLinks(candidate) { + const root = realpathSync(candidate); + for (const name of ENGINE_PACKAGES) { + const link = join(root, 'apps', 'cli', 'node_modules', '@relavium', name); + let info; + try { + info = lstatSync(link); + } catch { + assert.fail('PACKED_CLI_CANDIDATE_LINK'); + } + assert.ok(info.isSymbolicLink(), 'PACKED_CLI_CANDIDATE_LINK'); + assert.equal(realpathSync(link), join(root, 'packages', name), 'PACKED_CLI_CANDIDATE_LINK'); + } + for (const workspace of [ + join('apps', 'cli'), + ...ENGINE_PACKAGES.map((name) => join('packages', name)), + ]) { + const scope = join(root, workspace, 'node_modules', '@relavium'); + if (!existsSync(scope)) continue; + for (const name of readdirSync(scope)) { + assert.ok(ENGINE_PACKAGES.includes(name), 'PACKED_CLI_CANDIDATE_LINK'); + assert.equal( + realpathSync(join(scope, name)), + join(root, 'packages', name), + 'PACKED_CLI_CANDIDATE_LINK', + ); + } + } + assert.ok(within(root, realpathSync(join(root, 'node_modules'))), 'PACKED_CLI_CANDIDATE_LINK'); +} + +/** + * No build cache may exist in the candidate after its build: Turbo state, TypeScript incremental + * files or a tool cache would let a later build read candidate output or a broad capture keep it. + * Dependency directories themselves are installation state and are not walked. + * @param {string} candidate + */ +export function assertNoBuildCache(candidate) { + const found = []; + function visit(directory, prefix) { + for (const entry of readdirSync(directory, { withFileTypes: true })) { + const path = prefix === '' ? entry.name : `${prefix}/${entry.name}`; + if (entry.name === '.turbo' || entry.name.endsWith('.tsbuildinfo')) found.push(path); + else if (entry.isDirectory() && entry.name === 'node_modules') { + if (existsSync(join(directory, entry.name, '.cache'))) found.push(`${path}/.cache`); + } else if (entry.isDirectory()) visit(join(directory, entry.name), path); + } + } + visit(candidate, ''); + assert.deepEqual(found, [], 'PACKED_CLI_CANDIDATE_BUILD_CACHE'); +} + +/** + * Prepare the owned candidate: capture the original checkout's identity, copy its listed files, + * refuse if that identity changed during the copy, install the locked closure and verify every + * workspace link. The returned identity is what the candidate is later sealed with. + * @param {{repository: string, candidate: string, environment: Record, + * home: string, env: Record, excludes?: string, run?: typeof runCommand, + * pnpm?: typeof runPnpm}} options + * `env` is the isolated environment of the private `home`; `excludes` is the developer's global + * Git excludes file (`globalExcludesFile`). + */ +export async function prepareCandidate({ + repository, + candidate, + environment, + home, + env, + excludes = undefined, + run = runCommand, + pnpm = runPnpm, +}) { + const identity = await sourceIdentity(repository, run, excludes); + const copied = copyCandidateTree({ + repository, + destination: candidate, + files: await candidateFiles(repository, run, excludes), + }); + const after = await sourceIdentity(repository, run, excludes); + assert.ok( + after.source === identity.source && after.status === identity.status, + 'PACKED_CLI_SOURCE_CHANGED_DURING_COPY', + ); + const store = await dependencyStore({ repository, environment, home, pnpm }); + await installCandidateDependencies({ candidate, store, env, pnpm }); + verifyCandidateLinks(candidate); + return { source: identity.source, dirty: identity.dirty, files: copied.files }; +} diff --git a/tools/cli-package/candidate.test.mjs b/tools/cli-package/candidate.test.mjs new file mode 100644 index 00000000..3dd44786 --- /dev/null +++ b/tools/cli-package/candidate.test.mjs @@ -0,0 +1,578 @@ +import assert from 'node:assert/strict'; +import { + chmodSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + realpathSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { after, test } from 'node:test'; +import { + assertNoBuildCache, + candidateFiles, + copyCandidateTree, + dependencyStore, + globalExcludesFile, + installCandidateDependencies, + prepareCandidate, + sourceIdentity, + verifyCandidateLinks, +} from './candidate.mjs'; +import { isolatedEnvironment, runCommand } from './process.mjs'; + +const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-candidate-tree-'))); +after(() => rmSync(scratch, { recursive: true, force: true })); +const posixModes = process.platform !== 'win32'; +const linkKind = process.platform === 'win32' ? 'junction' : 'dir'; +let count = 0; + +function fresh(name) { + count += 1; + const path = join(scratch, `${name}-${count}`); + mkdirSync(path); + return path; +} + +function write(path, contents, mode) { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, contents); + if (mode !== undefined) chmodSync(path, mode); +} + +/** A fake `git` port answering from a table of exact argument lists. */ +function git(answers) { + const calls = []; + const run = async (command, args, options) => { + calls.push({ command, args, cwd: options.cwd, env: options.env }); + const key = args.join(' '); + assert.ok(Object.hasOwn(answers, key), `unexpected git ${key}`); + return { stdout: answers[key], stderr: '' }; + }; + return { run, calls }; +} + +const SHA = 'a'.repeat(40); + +test('source identity records the exact commit and the complete porcelain status', async (t) => { + // A credential in the caller's environment must not reach the isolated Git. + const previous = process.env.NPM_TOKEN; + process.env.NPM_TOKEN = 'synthetic-secret'; + t.after(() => { + if (previous === undefined) delete process.env.NPM_TOKEN; + else process.env.NPM_TOKEN = previous; + }); + const clean = git({ + '--no-replace-objects rev-parse HEAD': `${SHA}\n`, + 'status --porcelain=v1 -z --untracked-files=all': '', + }); + assert.deepEqual(await sourceIdentity('/repository', clean.run), { + source: SHA, + status: '', + dirty: false, + }); + assert.ok(clean.calls.every((call) => call.command === 'git' && call.cwd === '/repository')); + // The original checkout is queried with an isolated environment, never the caller's. + assert.ok( + clean.calls.every((call) => call.env.HOME === '/repository' && !('NPM_TOKEN' in call.env)), + ); + const modified = git({ + '--no-replace-objects rev-parse HEAD': `${SHA}\n`, + 'status --porcelain=v1 -z --untracked-files=all': '?? notes.txt\0', + }); + assert.equal((await sourceIdentity('/repository', modified.run)).dirty, true); + // The developer's global excludes file judges untracked files. + const excluded = git({ + '--no-replace-objects rev-parse HEAD': `${SHA}\n`, + '-c core.excludesFile=/home/someone/.config/git/ignore status --porcelain=v1 -z --untracked-files=all': + '', + }); + assert.equal( + (await sourceIdentity('/repository', excluded.run, '/home/someone/.config/git/ignore')).dirty, + false, + ); + for (const head of ['', 'HEAD\n', `${SHA.slice(1)}\n`, `${SHA}${SHA}\n`]) { + const malformed = git({ + '--no-replace-objects rev-parse HEAD': head, + 'status --porcelain=v1 -z --untracked-files=all': '', + }); + await assert.rejects(sourceIdentity('/repository', malformed.run), /SOURCE_IDENTITY/u); + } +}); + +test('candidate files are tracked plus unignored untracked files, minus deletions, sorted once', async () => { + const listing = git({ + 'ls-files -z --cached --others --exclude-standard': 'b.ts\0a.ts\0gone.ts\0a.ts\0new.ts\0', + 'ls-files -z --deleted': 'gone.ts\0', + }); + assert.deepEqual(await candidateFiles('/repository', listing.run), ['a.ts', 'b.ts', 'new.ts']); + const empty = git({ + 'ls-files -z --cached --others --exclude-standard': '', + 'ls-files -z --deleted': '', + }); + await assert.rejects(candidateFiles('/repository', empty.run), /CANDIDATE_FILES/u); + const excluded = git({ + '-c core.excludesFile=/ignore ls-files -z --cached --others --exclude-standard': 'a.ts\0', + 'ls-files -z --deleted': '', + }); + assert.deepEqual(await candidateFiles('/repository', excluded.run, '/ignore'), ['a.ts']); +}); + +test("the developer's global excludes file is located without the rest of the global configuration", async () => { + const home = fresh('developer-home'); + const ignore = join(home, '.config', 'git', 'ignore'); + write(ignore, 'secret.local\n'); + const calls = []; + const unset = async (command, args, options) => { + calls.push({ args, env: options.env }); + const error = new Error('PACKED_CLI_CHILD_FAILED: 1'); + error.result = { code: 1, stdout: '', stderr: '' }; + throw error; + }; + const environment = { + PATH: '/usr/bin', + HOME: home, + NPM_TOKEN: 'synthetic-secret', + NODE_OPTIONS: '--require=/tmp/preload.cjs', + GIT_DIR: '/elsewhere', + }; + assert.equal( + await globalExcludesFile({ repository: '/repository', environment, run: unset }), + ignore, + ); + assert.deepEqual(calls[0].args, ['config', '--path', '--get', 'core.excludesFile']); + assert.deepEqual(calls[0].env, { PATH: '/usr/bin', HOME: home }); + // Git's default prefers XDG_CONFIG_HOME, and a missing file means no global excludes. + const xdg = fresh('xdg'); + write(join(xdg, 'git', 'ignore'), '*.local\n'); + assert.equal( + await globalExcludesFile({ + repository: '/repository', + environment: { HOME: home, XDG_CONFIG_HOME: xdg }, + run: unset, + }), + join(xdg, 'git', 'ignore'), + ); + assert.equal( + await globalExcludesFile({ + repository: '/repository', + environment: { HOME: fresh('empty-home') }, + run: unset, + }), + undefined, + ); + // A configured core.excludesFile wins, resolved against the repository when relative. + const configured = join(fresh('configured'), 'ignore-file'); + write(configured, '*.secret\n'); + const answer = (stdout) => async () => ({ stdout, stderr: '' }); + assert.equal( + await globalExcludesFile({ + repository: '/repository', + environment: { HOME: home }, + run: answer(`${configured}\n`), + }), + configured, + ); + const repository = fresh('relative-repository'); + write(join(repository, 'local-ignore'), '*.secret\n'); + assert.equal( + await globalExcludesFile({ + repository, + environment: { HOME: home }, + run: answer('local-ignore\n'), + }), + join(repository, 'local-ignore'), + ); + // Any other Git failure refuses, and an explicit file from an isolating caller must exist. + const broken = async () => { + const error = new Error('PACKED_CLI_CHILD_FAILED: 128'); + error.result = { code: 128, stdout: '', stderr: '' }; + throw error; + }; + await assert.rejects( + globalExcludesFile({ repository: '/repository', environment: { HOME: home }, run: broken }), + /CHILD_FAILED/u, + ); + assert.equal( + await globalExcludesFile({ + repository: '/repository', + environment: { RELAVIUM_GIT_EXCLUDES_FILE: ignore }, + run: broken, + }), + ignore, + ); + for (const bad of ['relative/ignore', join(home, 'absent-ignore')]) { + await assert.rejects( + globalExcludesFile({ + repository: '/repository', + environment: { RELAVIUM_GIT_EXCLUDES_FILE: bad }, + run: broken, + }), + /GIT_EXCLUDES/u, + ); + } +}); + +test('a real checkout copies nothing its developer ignores globally', async () => { + const repository = fresh('real-checkout'); + const env = isolatedEnvironment(repository); + const inRepository = (args) => + runCommand('git', args, { cwd: repository, env, timeoutMs: 30_000 }); + await inRepository(['init', '--quiet']); + write(join(repository, 'kept.ts'), 'export {};\n'); + write(join(repository, 'secret.local'), 'private\n'); + await inRepository(['add', 'kept.ts']); + await inRepository([ + '-c', + 'user.name=t', + '-c', + 'user.email=t@example.invalid', + 'commit', + '--quiet', + '-m', + 'one', + ]); + const home = fresh('developer-home'); + write(join(home, '.config', 'git', 'ignore'), 'secret.local\n'); + const excludes = await globalExcludesFile({ + repository, + environment: { PATH: process.env.PATH, HOME: home }, + }); + assert.equal(excludes, join(home, '.config', 'git', 'ignore')); + assert.deepEqual(await candidateFiles(repository, runCommand, excludes), ['kept.ts']); + assert.equal((await sourceIdentity(repository, runCommand, excludes)).dirty, false); + // Without it the isolated Git sees the globally ignored file as untracked. + assert.deepEqual(await candidateFiles(repository), ['kept.ts', 'secret.local']); + assert.equal((await sourceIdentity(repository)).dirty, true); +}); + +test('the candidate copy is byte-exact, keeps only the executable bit and refuses links', () => { + const repository = fresh('repository'); + write(join(repository, 'package.json'), '{"name":"root"}\n'); + write(join(repository, 'tools', 'run.mjs'), '#!/usr/bin/env node\n', 0o775); + write(join(repository, 'docs', 'note.md'), 'note\n', 0o640); + const destination = fresh('copy'); + const files = ['docs/note.md', 'package.json', 'tools/run.mjs']; + assert.deepEqual(copyCandidateTree({ repository, destination, files }), { files: 3, bytes: 41 }); + for (const path of files) { + assert.ok(readFileSync(join(destination, path)).equals(readFileSync(join(repository, path)))); + } + if (posixModes) { + assert.equal(statSync(join(destination, 'tools', 'run.mjs')).mode & 0o777, 0o755); + assert.equal(statSync(join(destination, 'docs', 'note.md')).mode & 0o777, 0o644); + } + if (posixModes) { + // The modes are exact whatever the process umask. + const previous = process.umask(0o077); + let restrictive; + try { + restrictive = fresh('copy'); + copyCandidateTree({ repository, destination: restrictive, files }); + } finally { + process.umask(previous); + } + assert.equal(statSync(join(restrictive, 'tools', 'run.mjs')).mode & 0o777, 0o755); + assert.equal(statSync(join(restrictive, 'docs', 'note.md')).mode & 0o777, 0o644); + } + assert.throws( + () => copyCandidateTree({ repository, destination, files }), + /CANDIDATE_ROOT/u, + 'a non-empty destination refuses', + ); + for (const path of ['../escape.ts', 'a/../b.ts', './a.ts', 'a//b.ts', 'a\\b.ts', '']) { + assert.throws( + () => copyCandidateTree({ repository, destination: fresh('copy'), files: [path] }), + /CANDIDATE_PATH/u, + ); + } + if (posixModes) { + symlinkSync(join(repository, 'package.json'), join(repository, 'linked.json')); + assert.throws( + () => copyCandidateTree({ repository, destination: fresh('copy'), files: ['linked.json'] }), + /FILE_KIND_OR_BOUND/u, + ); + } +}); + +test('the dependency store is discovered with store-location variables only', async () => { + const store = fresh('store'); + const calls = []; + const pnpm = async (args, options) => { + calls.push({ args, options }); + return { stdout: `${store}\n`, stderr: '' }; + }; + const environment = { + HOME: '/home/someone', + XDG_DATA_HOME: '/home/someone/.local/share', + NPM_TOKEN: 'synthetic-secret', + NODE_OPTIONS: '--require=/tmp/preload.cjs', + npm_config_registry: 'https://registry.invalid/', + }; + const home = fresh('private-home'); + assert.equal( + await dependencyStore({ repository: '/repository', environment, home, pnpm }), + store, + ); + assert.deepEqual(calls[0].args, ['store', 'path']); + const env = calls[0].options.env; + assert.equal(env.HOME, '/home/someone'); + assert.equal(env.XDG_DATA_HOME, '/home/someone/.local/share'); + // Temporary files and caches (Node's compile cache, corepack) never land in the checkout. + for (const key of ['TMPDIR', 'TMP', 'TEMP', 'XDG_CACHE_HOME']) assert.equal(env[key], home, key); + for (const key of [ + 'NPM_TOKEN', + 'NODE_OPTIONS', + 'npm_config_registry', + 'USERPROFILE', + 'XDG_CONFIG_HOME', + ]) { + assert.equal(key in env, false, key); + } + // A caller that already isolated this process names the store explicitly; pnpm is not asked. + const explicit = fresh('explicit-store'); + assert.equal( + await dependencyStore({ + repository: '/repository', + environment: { RELAVIUM_PACKAGE_STORE_DIR: explicit }, + home, + pnpm: async () => { + throw new Error('store discovery must not run'); + }, + }), + explicit, + ); + for (const bad of ['relative/store', join(scratch, 'absent-store')]) { + await assert.rejects( + dependencyStore({ + repository: '/repository', + environment: { RELAVIUM_PACKAGE_STORE_DIR: bad }, + home, + pnpm, + }), + /DEPENDENCY_STORE/u, + ); + } + for (const answer of ['relative/store\n', `${join(scratch, 'absent-store')}\n`]) { + await assert.rejects( + dependencyStore({ + repository: '/repository', + environment, + home, + pnpm: async () => ({ stdout: answer, stderr: '' }), + }), + /DEPENDENCY_STORE/u, + ); + } +}); + +test('store discovery refuses a private home inside the original checkout', async () => { + const repository = fresh('checkout'); + await assert.rejects( + dependencyStore({ + repository, + environment: { HOME: '/home/someone' }, + home: join(repository, 'home'), + pnpm: async () => { + throw new Error('store discovery must not run'); + }, + }), + /PRIVATE_HOME/u, + ); +}); + +test('the candidate installs offline, frozen and without lifecycle scripts', async () => { + const calls = []; + await installCandidateDependencies({ + candidate: '/candidate', + store: '/store', + env: { HOME: '/home' }, + pnpm: async (args, options) => { + calls.push({ args, options }); + return { stdout: '', stderr: '' }; + }, + }); + assert.deepEqual(calls, [ + { + args: [ + 'install', + '--offline', + '--frozen-lockfile', + '--ignore-scripts', + '--store-dir', + '/store', + ], + options: { cwd: '/candidate', env: { HOME: '/home' } }, + }, + ]); +}); + +function linkedCandidate() { + const root = fresh('candidate'); + mkdirSync(join(root, 'node_modules')); + for (const name of ['shared', 'llm', 'core', 'db', 'mcp']) { + mkdirSync(join(root, 'packages', name), { recursive: true }); + } + const scope = join(root, 'apps', 'cli', 'node_modules', '@relavium'); + mkdirSync(scope, { recursive: true }); + for (const name of ['shared', 'llm', 'core', 'db', 'mcp']) { + symlinkSync(join(root, 'packages', name), join(scope, name), linkKind); + } + return root; +} + +test('every workspace link must resolve inside the candidate', () => { + const root = linkedCandidate(); + verifyCandidateLinks(root); + const original = fresh('original-checkout'); + mkdirSync(join(original, 'core')); + const link = join(root, 'apps', 'cli', 'node_modules', '@relavium', 'core'); + rmSync(link); + symlinkSync(join(original, 'core'), link, linkKind); + assert.throws(() => verifyCandidateLinks(root), /CANDIDATE_LINK/u); + + const nested = linkedCandidate(); + const coreScope = join(nested, 'packages', 'core', 'node_modules', '@relavium'); + mkdirSync(coreScope, { recursive: true }); + symlinkSync(join(original, 'core'), join(coreScope, 'shared'), linkKind); + assert.throws(() => verifyCandidateLinks(nested), /CANDIDATE_LINK/u); + + const missing = linkedCandidate(); + rmSync(join(missing, 'apps', 'cli', 'node_modules', '@relavium', 'mcp')); + assert.throws(() => verifyCandidateLinks(missing), /CANDIDATE_LINK/u); + + // The root node_modules must not resolve into the original checkout either. + const borrowed = linkedCandidate(); + rmSync(join(borrowed, 'node_modules'), { recursive: true }); + mkdirSync(join(original, 'node_modules')); + symlinkSync(join(original, 'node_modules'), join(borrowed, 'node_modules'), linkKind); + assert.throws(() => verifyCandidateLinks(borrowed), /CANDIDATE_LINK/u); +}); + +test('a candidate build leaves no build cache behind', () => { + const root = fresh('built'); + write(join(root, 'packages', 'core', 'dist', 'index.js'), 'export {};\n'); + write(join(root, 'node_modules', 'left-pad', 'index.js'), 'module.exports = 1;\n'); + assertNoBuildCache(root); + for (const marker of [ + 'packages/core/tsconfig.build.tsbuildinfo', + '.turbo/cache/abc.tar.zst', + 'apps/cli/.turbo/turbo-build.log', + 'node_modules/.cache/tool/state', + 'apps/cli/node_modules/.cache/state', + ]) { + const candidate = fresh('built'); + write(join(candidate, 'packages', 'core', 'dist', 'index.js'), 'export {};\n'); + write(join(candidate, marker), 'cache\n'); + assert.throws(() => assertNoBuildCache(candidate), /CANDIDATE_BUILD_CACHE/u, marker); + } + assert.deepEqual(readdirSync(root).sort(), ['node_modules', 'packages']); +}); + +/** A source repository plus fake git and pnpm ports; the fake install creates the workspace links. */ +function preparation({ statusAfterCopy = '', headAfterCopy = SHA } = {}) { + const repository = fresh('source'); + write(join(repository, 'package.json'), '{}\n'); + for (const name of ['shared', 'llm', 'core', 'db', 'mcp']) { + write(join(repository, 'packages', name, 'package.json'), '{}\n'); + } + const files = [ + 'package.json', + ...['shared', 'llm', 'core', 'db', 'mcp'].map((name) => `packages/${name}/package.json`), + ]; + const candidate = fresh('candidate'); + const store = fresh('store'); + const home = fresh('home'); + const events = []; + let statuses = 0; + let heads = 0; + const run = async (command, args) => { + const key = args.join(' '); + events.push(`git ${key}`); + if (key === '--no-replace-objects rev-parse HEAD') { + heads += 1; + return { stdout: `${heads === 1 ? SHA : headAfterCopy}\n` }; + } + if (key === 'status --porcelain=v1 -z --untracked-files=all') { + statuses += 1; + return { stdout: statuses === 1 ? '' : statusAfterCopy }; + } + if (key === 'ls-files -z --cached --others --exclude-standard') + return { stdout: files.join('\0') }; + if (key === 'ls-files -z --deleted') return { stdout: '' }; + throw new Error(`unexpected git ${key}`); + }; + const pnpm = async (args, options) => { + events.push(`pnpm ${args.join(' ')}`); + if (args[0] === 'store') return { stdout: `${store}\n` }; + assert.equal(options.cwd, candidate); + mkdirSync(join(candidate, 'node_modules'), { recursive: true }); + const scope = join(candidate, 'apps', 'cli', 'node_modules', '@relavium'); + mkdirSync(scope, { recursive: true }); + for (const name of ['shared', 'llm', 'core', 'db', 'mcp']) { + symlinkSync(join(candidate, 'packages', name), join(scope, name), linkKind); + } + return { stdout: '' }; + }; + return { repository, candidate, store, home, events, run, pnpm }; +} + +test('candidate preparation copies, re-checks identity, installs and verifies links in order', async () => { + const ports = preparation(); + const identity = await prepareCandidate({ + repository: ports.repository, + candidate: ports.candidate, + environment: { HOME: '/home/someone' }, + home: ports.home, + env: { HOME: '/isolated' }, + run: ports.run, + pnpm: ports.pnpm, + }); + assert.deepEqual(identity, { source: SHA, dirty: false, files: 6 }); + assert.deepEqual(ports.events, [ + 'git --no-replace-objects rev-parse HEAD', + 'git status --porcelain=v1 -z --untracked-files=all', + 'git ls-files -z --cached --others --exclude-standard', + 'git ls-files -z --deleted', + 'git --no-replace-objects rev-parse HEAD', + 'git status --porcelain=v1 -z --untracked-files=all', + 'pnpm store path', + `pnpm install --offline --frozen-lockfile --ignore-scripts --store-dir ${ports.store}`, + ]); + assert.ok( + readFileSync(join(ports.candidate, 'package.json')).equals( + readFileSync(join(ports.repository, 'package.json')), + ), + ); +}); + +test('a source change during the copy refuses before anything is installed', async () => { + // A changed status, or a commit that moved while the status stayed the same. + for (const change of [ + { statusAfterCopy: ' M package.json\0' }, + { headAfterCopy: 'b'.repeat(40) }, + ]) { + const ports = preparation(change); + await assert.rejects( + prepareCandidate({ + repository: ports.repository, + candidate: ports.candidate, + environment: {}, + home: ports.home, + env: {}, + run: ports.run, + pnpm: ports.pnpm, + }), + /SOURCE_CHANGED_DURING_COPY/u, + ); + assert.equal( + ports.events.some((event) => event.startsWith('pnpm')), + false, + ); + } +}); diff --git a/tools/cli-package/check.mjs b/tools/cli-package/check.mjs index 3aaba4dd..1db2bfd8 100644 --- a/tools/cli-package/check.mjs +++ b/tools/cli-package/check.mjs @@ -4,16 +4,34 @@ import { mkdtempSync, readFileSync, realpathSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { + GIT_EXCLUDES_ENV, + PACKAGE_STORE_ENV, + dependencyStore, + globalExcludesFile, +} from './candidate.mjs'; import { isolatedEnvironment, runCommand } from './process.mjs'; assert.equal(process.argv.length, 2, 'no arguments required'); const directory = fileURLToPath(new URL('.', import.meta.url)); const root = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-local-package-check-'))); try { + // The producer runs with an isolated HOME, so the store and the developer's global Git excludes + // file are found here, from the caller's own location variables, and handed over explicitly. + const repository = fileURLToPath(new URL('../../', import.meta.url)); + const store = await dependencyStore({ repository, environment: process.env, home: root }); + const excludes = await globalExcludesFile({ repository, environment: process.env }); const artifact = join(root, 'artifact'), reports = join(root, 'reports'); const options = { env: isolatedEnvironment(root), timeoutMs: 600_000 }; - await runCommand(process.execPath, [join(directory, 'pack.mjs'), artifact], options); + await runCommand(process.execPath, [join(directory, 'pack.mjs'), artifact], { + ...options, + env: { + ...options.env, + [PACKAGE_STORE_ENV]: store, + ...(excludes === undefined ? {} : { [GIT_EXCLUDES_ENV]: excludes }), + }, + }); await runCommand(process.execPath, [join(directory, 'consume.mjs'), artifact, reports], options); const record = JSON.parse(readFileSync(join(reports, 'installed-consumer.json'), 'utf8')); assert.equal(record.leg, 'local'); diff --git a/tools/cli-package/finalize.mjs b/tools/cli-package/finalize.mjs new file mode 100644 index 00000000..9eab4723 --- /dev/null +++ b/tools/cli-package/finalize.mjs @@ -0,0 +1,462 @@ +/** + * Candidate finalizer (ADR-0111, Step 9a.1). It runs inside the CLI's prepack after the sole + * candidate tsup build and before pnpm packs. It accounts for every emitted output, validates every + * consumed upstream TypeScript map and every final map, rewrites final map sources to + * repository-relative original TypeScript, removes only tsup's exact terminal map-link trailers, + * stages the maps and the build metadata owner-only outside the package, re-checks the finite + * closure and the public file/mode inventory, and only then hashes. Nothing writes JS afterwards. + * + * Refusals name a fixed code and at most a repository-relative output or source path, never map or + * source contents. A successful bundle is no evidence by itself: a missing upstream map makes the + * bundler silently map engine frames to compiled JS, so every source set is checked explicitly. + */ +import { Buffer } from 'node:buffer'; +import { + closeSync, + fchmodSync, + fsyncSync, + lstatSync, + mkdirSync, + openSync, + readdirSync, + renameSync, + unlinkSync, + writeSync, +} from 'node:fs'; +import { basename, isAbsolute, join, posix, relative } from 'node:path'; +import { TextDecoder, isDeepStrictEqual } from 'node:util'; +import { checkBundleClosure } from '../bundle-closure/closure.mjs'; +import { MODE_EVIDENCE, emittedInventory } from '../bundle-closure/inventory.mjs'; +import { flatFiles, readOrdinaryFile, sha256 } from './files.mjs'; + +/** + * Names the private staging directory of a trusted candidate build. Only the package producer sets + * it; its presence alone selects candidate maps in the CLI build and the finalizer in prepack. + */ +export const CANDIDATE_STAGING_ENV = 'RELAVIUM_CLI_CANDIDATE_STAGING'; + +/** The inlined engine packages, in their dependency build order. */ +export const ENGINE_PACKAGES = Object.freeze(['shared', 'llm', 'core', 'db', 'mcp']); +const LIMITS = Object.freeze({ + metafileBytes: 4 * 1024 * 1024, + jsBytes: 8 * 1024 * 1024, + upstreamMapBytes: 4 * 1024 * 1024, + finalMapBytes: 16 * 1024 * 1024, + outputs: 256, + sourcesPerMap: 4096, + namesPerMap: 1_000_000, +}); +const SOURCE_TAIL = /^(?:[A-Za-z0-9_-][A-Za-z0-9_.-]*\/)*[A-Za-z0-9_-][A-Za-z0-9_.-]*\.tsx?$/u; +const MAPPINGS = /^[A-Za-z0-9+/,;]*$/u; + +export class CandidateFinalizationError extends Error { + /** @param {string} code @param {string} [subject] a repository-relative path, never contents */ + constructor(code, subject) { + super( + `CLI candidate finalization refused: ${code}` + + (subject === undefined ? '' : ` (${subject})`), + ); + this.name = 'CandidateFinalizationError'; + this.code = code; + } +} + +/** @param {string} code @param {string} [subject] @returns {never} */ +function refuse(code, subject) { + throw new CandidateFinalizationError(code, subject); +} + +/** @param {unknown} value @returns {value is Record} */ +function isRecord(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +/** The map-link trailer tsc and tsup write for an external map beside its JS file. */ +export const mapTrailer = (/** @type {string} */ jsName) => `//# sourceMappingURL=${jsName}.map`; + +/** @param {string} parent @param {string} child */ +function within(parent, child) { + const path = relative(parent, child); + return path === '' || (!path.startsWith('..') && !isAbsolute(path)); +} + +/** @param {string} root @param {string} path repository-relative POSIX path */ +function onDisk(root, path) { + return join(root, ...path.split('/')); +} + +/** @param {string} path @param {number} limit @param {string} subject */ +function readBytes(path, limit, subject) { + try { + return readOrdinaryFile(path, limit); + } catch { + refuse('UNREADABLE_FILE', subject); + } +} + +/** @param {Uint8Array} bytes @param {string} subject */ +function parseJson(bytes, subject) { + try { + return JSON.parse(new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(bytes)); + } catch { + refuse('MALFORMED_JSON', subject); + } +} + +/** @param {string} path @param {number} limit @param {string} subject */ +function readJson(path, limit, subject) { + return parseJson(readBytes(path, limit, subject), subject); +} + +/** @param {string} path @param {string} subject */ +function readText(path, subject) { + try { + return new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode( + readOrdinaryFile(path, LIMITS.jsBytes), + ); + } catch { + refuse('UNREADABLE_FILE', subject); + } +} + +/** + * Resolve one map source against its map's repository-relative directory and admit only original + * TypeScript inside `allowedRoots`. URLs, absolute or escaping paths and anything but a `.ts`/`.tsx` + * source refuse — a JS source means the bundler fell back to compiled output. + * @param {string} repository + * @param {string} mapDirectory repository-relative POSIX directory of the map file + * @param {unknown} source + * @param {readonly string[]} allowedRoots repository-relative POSIX source roots + * @param {string} subject + */ +function originalSource(repository, mapDirectory, source, allowedRoots, subject) { + if ( + typeof source !== 'string' || + source.length === 0 || + source.length > 1024 || + /[\\%?#]/u.test(source) || + [...source].some((character) => (character.codePointAt(0) ?? 0) < 32) || + source.startsWith('/') || + /^[A-Za-z][A-Za-z0-9+.-]*:/u.test(source) + ) { + refuse('MAP_SOURCE_PATH', subject); + } + const resolved = posix.normalize(posix.join(mapDirectory, source)); + const root = allowedRoots.find((candidate) => resolved.startsWith(`${candidate}/`)); + if ( + root === undefined || + resolved.endsWith('.d.ts') || + !SOURCE_TAIL.test(resolved.slice(root.length + 1)) + ) { + refuse('MAP_SOURCE_NOT_ORIGINAL', subject); + } + let info; + try { + info = lstatSync(onDisk(repository, resolved)); + } catch { + refuse('MAP_SOURCE_MISSING', subject); + } + if (!info.isFile()) refuse('MAP_SOURCE_MISSING', subject); + return resolved; +} + +/** + * Validate one consumed upstream (tsc) map. The JS must end with its exact trailer and the map must + * name only this package's original TypeScript, without embedded sources or a source root. + * @param {string} repository + * @param {string} packageName + * @param {string} jsPath repository-relative POSIX path of the consumed compiled JS + */ +function upstreamSources(repository, packageName, jsPath) { + const name = posix.basename(jsPath); + const js = readText(onDisk(repository, jsPath), jsPath); + if (!js.endsWith(mapTrailer(name)) || js.split('sourceMappingURL').length !== 2) { + refuse('UPSTREAM_MAP_LINK', jsPath); + } + const mapPath = `${jsPath}.map`; + let exists = false; + try { + exists = lstatSync(onDisk(repository, mapPath)).isFile(); + } catch { + exists = false; + } + if (!exists) refuse('UPSTREAM_MAP_MISSING', jsPath); + const map = readJson(onDisk(repository, mapPath), LIMITS.upstreamMapBytes, mapPath); + if ( + !isRecord(map) || + !isDeepStrictEqual(Object.keys(map).sort(), [ + 'file', + 'mappings', + 'names', + 'sourceRoot', + 'sources', + 'version', + ]) || + map.version !== 3 || + map.file !== name || + map.sourceRoot !== '' || + typeof map.mappings !== 'string' || + !MAPPINGS.test(map.mappings) || + !Array.isArray(map.names) || + map.names.some((item) => typeof item !== 'string') || + !Array.isArray(map.sources) || + map.sources.length === 0 || + map.sources.length > LIMITS.sourcesPerMap + ) { + refuse('UPSTREAM_MAP_SHAPE', mapPath); + } + const directory = posix.dirname(jsPath); + return map.sources.map((source) => + originalSource(repository, directory, source, [`packages/${packageName}/src`], mapPath), + ); +} + +/** + * Original TypeScript sources behind one metafile input: the CLI's own source directly, an engine + * package's compiled JS through its tsc map. Any other input kind refuses. + * @param {string} repository @param {string} input metafile input relative to apps/cli + * @param {Map} cache + */ +function inputSources(repository, input, cache) { + const cached = cache.get(input); + if (cached) return cached; + let sources; + const engine = /^\.\.\/\.\.\/packages\/([a-z]+)\/dist\/(.+\.js)$/u.exec(input); + if (input.startsWith('src/') && SOURCE_TAIL.test(input.slice(4)) && !input.endsWith('.d.ts')) { + sources = [originalSource(repository, 'apps/cli', input, ['apps/cli/src'], input)]; + } else if ( + engine && + ENGINE_PACKAGES.includes(engine[1]) && + /^(?:[A-Za-z0-9_-][A-Za-z0-9_.-]*\/)*[A-Za-z0-9_-][A-Za-z0-9_.-]*\.js$/u.test(engine[2]) + ) { + sources = upstreamSources(repository, engine[1], `packages/${engine[1]}/dist/${engine[2]}`); + } else { + refuse('UNSUPPORTED_INPUT', input); + } + cache.set(input, sources); + return sources; +} + +/** + * Validate one final map against the original sources of the inputs that contributed bytes to its + * runtime output, and return it with repository-relative sources. Mappings and names are kept. + * @param {string} repository @param {string} jsName @param {unknown} map @param {Set} expected + */ +function finalMap(repository, jsName, map, expected) { + const subject = `apps/cli/dist/${jsName}.map`; + const keys = isRecord(map) ? Object.keys(map) : []; + if ( + !isRecord(map) || + !['mappings', 'names', 'sources', 'version'].every((key) => keys.includes(key)) || + keys.some( + (key) => !['file', 'mappings', 'names', 'sourceRoot', 'sources', 'version'].includes(key), + ) + ) { + // sourcesContent, an index map's sections or any unreviewed field refuses. + refuse(isRecord(map) && 'sourcesContent' in map ? 'EMBEDDED_SOURCES' : 'MAP_SHAPE', subject); + } + if ( + map.version !== 3 || + (map.file !== undefined && map.file !== jsName) || + (map.sourceRoot !== undefined && map.sourceRoot !== '') || + typeof map.mappings !== 'string' || + !MAPPINGS.test(map.mappings) || + !Array.isArray(map.names) || + map.names.length > LIMITS.namesPerMap || + map.names.some((item) => typeof item !== 'string') || + !Array.isArray(map.sources) || + map.sources.length > LIMITS.sourcesPerMap || + // A code-splitting facade that only re-exports other chunks has no contributing input and an + // empty map; any output with contributing inputs must map them. + (expected.size === 0 + ? map.sources.length !== 0 || map.mappings !== '' || map.names.length !== 0 + : map.mappings.length === 0) + ) { + refuse('MAP_SHAPE', subject); + } + const sources = map.sources.map((source) => + originalSource( + repository, + 'apps/cli/dist', + source, + ['apps/cli/src', ...ENGINE_PACKAGES.map((name) => `packages/${name}/src`)], + subject, + ), + ); + if (new Set(sources).size !== sources.length) refuse('MAP_SOURCE_DUPLICATE', subject); + if (!isDeepStrictEqual([...sources].sort(), [...expected].sort())) { + refuse('MAP_SOURCE_COVERAGE', subject); + } + return { ...map, sources }; +} + +/** + * A private staging directory must be absolute, empty, an ordinary directory outside the candidate + * repository and, where POSIX modes are observable, owner-only and owned by this process. + * @param {string} staging @param {string} repository + */ +export function assertPrivateStaging(staging, repository) { + if (typeof staging !== 'string' || !isAbsolute(staging)) refuse('STAGING_PATH'); + let info; + try { + info = lstatSync(staging); + } catch { + refuse('STAGING_PATH'); + } + if (!info.isDirectory() || within(repository, staging) || within(staging, repository)) { + refuse('STAGING_PATH'); + } + if (readdirSync(staging).length !== 0) refuse('STAGING_NOT_EMPTY'); + if ( + MODE_EVIDENCE === 'posix' && + ((info.mode & 0o777) !== 0o700 || + (typeof process.getuid === 'function' && info.uid !== process.getuid())) + ) { + refuse('STAGING_NOT_PRIVATE'); + } +} + +/** Create a new file with an exact mode (independent of the umask) and flush its bytes. */ +function writeNew(path, bytes, mode) { + const descriptor = openSync(path, 'wx', mode); + try { + if (MODE_EVIDENCE === 'posix') fchmodSync(descriptor, mode); + let offset = 0; + while (offset < bytes.length) offset += writeSync(descriptor, bytes, offset); + fsyncSync(descriptor); + } finally { + closeSync(descriptor); + } +} + +/** Write bytes to a new owner-only staging file. */ +function writePrivate(path, bytes) { + writeNew(path, bytes, 0o600); +} + +/** Replace one runtime output in place without moving any executable line or column. */ +function replaceOutput(path, bytes, mode) { + const temporary = `${path}.finalizing`; + writeNew(temporary, bytes, mode); + renameSync(temporary, path); +} + +/** + * Finalize the candidate CLI build in `/apps/cli` into `staging`. + * @param {{repository: string, staging: string}} options absolute candidate root and private staging + * @returns {{outputs: {path: string, bytes: number, sha256: string}[], maps: {path: string, bytes: number, sha256: string}[], upstreamMaps: number}} + * @throws {CandidateFinalizationError} for any unaccounted, unchained or malformed output. + */ +export function finalizeCandidate({ repository, staging }) { + assertPrivateStaging(staging, repository); + const cli = join(repository, 'apps', 'cli'); + const dist = join(cli, 'dist'); + const metafileBytes = readBytes( + join(dist, 'metafile-esm.json'), + LIMITS.metafileBytes, + 'metafile', + ); + const metafile = parseJson(metafileBytes, 'metafile'); + if (!isRecord(metafile) || !isRecord(metafile.outputs) || !isRecord(metafile.inputs)) { + refuse('METAFILE_SHAPE'); + } + + // 1. Every emitted file is a paired JS output and map, or the metafile. Nothing else. + const runtime = Object.keys(metafile.outputs).filter((path) => !path.endsWith('.map')); + if (runtime.length === 0 || runtime.length > LIMITS.outputs) refuse('OUTPUT_INVENTORY'); + const expectedEntries = ['metafile-esm.json']; + for (const path of runtime) { + if (!/^dist\/[A-Za-z0-9_-]+\.js$/u.test(path)) refuse('OUTPUT_INVENTORY', `apps/cli/${path}`); + if (!Object.hasOwn(metafile.outputs, `${path}.map`)) refuse('MAP_PAIRING', `apps/cli/${path}`); + expectedEntries.push(basename(path), `${basename(path)}.map`); + } + if (Object.keys(metafile.outputs).length !== runtime.length * 2) refuse('MAP_PAIRING'); + const entries = readdirSync(dist, { withFileTypes: true }); + if ( + entries.some((entry) => !entry.isFile()) || + !isDeepStrictEqual(entries.map((entry) => entry.name).sort(), expectedEntries.sort()) + ) { + refuse('OUTPUT_INVENTORY'); + } + + // 2. Validate every consumed upstream map and every final map, and strip each exact trailer. + // No file is written until every output has been accepted. + const cache = new Map(); + for (const input of Object.keys(metafile.inputs)) inputSources(repository, input, cache); + const prepared = runtime.map((path) => { + const jsName = basename(path); + const output = metafile.outputs[path]; + if (!isRecord(output) || !isRecord(output.inputs) || !Number.isSafeInteger(output.bytes)) { + refuse('METAFILE_SHAPE', `apps/cli/${path}`); + } + const js = readText(join(dist, jsName), `apps/cli/${path}`); + const trailer = mapTrailer(jsName); + if (!js.endsWith(trailer) || js.split('sourceMappingURL').length !== 2) { + refuse('FINAL_MAP_LINK', `apps/cli/${path}`); + } + const code = Buffer.from(js.slice(0, -trailer.length), 'utf8'); + // tsup appends the trailer after esbuild records the output size, so only the trailer may differ. + if (code.length !== output.bytes) refuse('TRAILER_BYTES', `apps/cli/${path}`); + const expected = new Set(); + for (const [input, contribution] of Object.entries(output.inputs)) { + if (!isRecord(contribution) || !Number.isSafeInteger(contribution.bytesInOutput)) { + refuse('METAFILE_SHAPE', `apps/cli/${path}`); + } + if (contribution.bytesInOutput === 0) continue; + const sources = cache.get(input); + if (!sources) refuse('UNSUPPORTED_INPUT', input); + for (const source of sources) expected.add(source); + } + const map = finalMap( + repository, + jsName, + readJson(join(dist, `${jsName}.map`), LIMITS.finalMapBytes, `apps/cli/${path}.map`), + expected, + ); + const mode = lstatSync(join(dist, jsName)).mode & 0o777; + return { path, jsName, code, map: Buffer.from(JSON.stringify(map), 'utf8'), mode }; + }); + + // 3. Stage maps and build metadata owner-only outside the package, then finalize the JS. + mkdirSync(join(staging, 'maps'), { mode: 0o700 }); + for (const item of prepared) writePrivate(join(staging, 'maps', `${item.jsName}.map`), item.map); + writePrivate(join(staging, 'metafile-esm.json'), metafileBytes); + for (const item of prepared) replaceOutput(join(dist, item.jsName), item.code, item.mode); + for (const item of prepared) unlinkSync(join(dist, `${item.jsName}.map`)); + unlinkSync(join(dist, 'metafile-esm.json')); + + // 4. Re-check the finite closure and the public inventory on the finalized files, then hash. + const manifest = readJson(join(cli, 'package.json'), 128 * 1024, 'apps/cli/package.json'); + checkBundleClosure({ + metafile, + manifest, + files: emittedInventory(cli), + modeEvidence: MODE_EVIDENCE, + maps: 'staged', + }); + if ( + !isDeepStrictEqual( + [...flatFiles(join(cli, 'drizzle'), 'package/drizzle')], + [...flatFiles(join(repository, 'packages', 'db', 'drizzle'), 'package/drizzle')], + ) + ) { + refuse('MIGRATION_COPY'); + } + const outputs = prepared.map((item) => { + const bytes = readOrdinaryFile(join(dist, item.jsName), LIMITS.jsBytes); + if (!bytes.equals(item.code)) refuse('OUTPUT_CHANGED', `apps/cli/${item.path}`); + return { path: item.path, bytes: bytes.length, sha256: sha256(bytes) }; + }); + const maps = prepared.map((item) => ({ + path: `${item.path}.map`, + bytes: item.map.length, + sha256: sha256(item.map), + })); + const upstreamMaps = [...cache.keys()].filter((input) => !input.startsWith('src/')).length; + const record = { version: 1, outputs, maps, upstreamMaps }; + writePrivate( + join(staging, 'finalized.json'), + Buffer.from(JSON.stringify(record, null, 2) + '\n'), + ); + return record; +} diff --git a/tools/cli-package/finalize.test.mjs b/tools/cli-package/finalize.test.mjs new file mode 100644 index 00000000..df9e96d2 --- /dev/null +++ b/tools/cli-package/finalize.test.mjs @@ -0,0 +1,478 @@ +import assert from 'node:assert/strict'; +import { + chmodSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + realpathSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import { basename, dirname, join, relative, sep } from 'node:path'; +import { after, test } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import ts from 'typescript'; +import { BundleClosureError } from '../bundle-closure/closure.mjs'; +import { CandidateFinalizationError, finalizeCandidate, mapTrailer } from './finalize.mjs'; + +// The exact esbuild behind the installed tsup and the installed TypeScript compiler produce real +// upstream and final maps from tiny fixtures; tsup's writer is reproduced exactly (trailer appended +// to the chunk code, map JSON re-serialized beside it). No product build runs here. +const cliRequire = createRequire(new URL('../../apps/cli/package.json', import.meta.url)); +const { build } = createRequire(cliRequire.resolve('tsup/package.json'))('esbuild'); +const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-candidate-finalizer-'))); +after(() => rmSync(scratch, { recursive: true, force: true })); +const posixModes = process.platform !== 'win32'; +let count = 0; + +function write(path, contents, mode) { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, contents); + if (mode !== undefined) chmodSync(path, mode); +} + +/** Every file below a directory, with its bytes, for unchanged-state assertions. */ +function snapshot(directory) { + const result = {}; + for (const entry of readdirSync(directory, { withFileTypes: true, recursive: true })) { + if (!entry.isFile()) continue; + const path = join(entry.parentPath ?? entry.path, entry.name); + result[relative(directory, path).split(sep).join('/')] = readFileSync(path).toString('base64'); + } + return result; +} + +/** + * A candidate repository whose CLI imports an engine package through its compiled JS, as the + * product does, built with splitting so it has shared chunks and both kinds of code-splitting + * facade: a statically and dynamically imported module (no inputs) and a pure re-export module + * (a zero-byte input). + */ +async function fixture({ upstreamMapAtBundle = true, sourcesContent = false } = {}) { + count += 1; + const root = join(scratch, `repository-${count}`); + const core = join(root, 'packages', 'core'); + const cli = join(root, 'apps', 'cli'); + write( + join(core, 'src', 'index.ts'), + "export * from './types.js';\nexport const greet = (name: string): string => `hello ${name}`;\n", + ); + write(join(core, 'src', 'types.ts'), 'export interface Shape {\n size: number;\n}\n'); + const program = ts.createProgram([join(core, 'src', 'index.ts'), join(core, 'src', 'types.ts')], { + rootDir: join(core, 'src'), + outDir: join(core, 'dist'), + sourceMap: true, + inlineSources: false, + module: ts.ModuleKind.ESNext, + target: ts.ScriptTarget.ES2022, + moduleResolution: ts.ModuleResolutionKind.Bundler, + strict: true, + skipLibCheck: true, + types: [], + }); + assert.equal(program.emit().emitSkipped, false); + const upstreamMap = join(core, 'dist', 'index.js.map'); + const upstreamMapBytes = readFileSync(upstreamMap); + if (!upstreamMapAtBundle) rmSync(upstreamMap); + write(join(root, 'packages', 'db', 'drizzle', '0000_init.sql'), 'create table t (id integer);\n'); + write(join(cli, 'drizzle', '0000_init.sql'), 'create table t (id integer);\n'); + write( + join(cli, 'package.json'), + JSON.stringify({ name: 'relavium', version: '0.0.0', type: 'module', dependencies: {} }), + ); + write( + join(cli, 'src', 'index.ts'), + "#!/usr/bin/env node\nimport { greet } from '../../../packages/core/dist/index.js';\n" + + "import { later } from './lazy.js';\nconsole.log(greet('cli'), later());\n" + + "export const lazy = () => import('./lazy.js');\nexport const facade = () => import('./facade.js');\n", + ); + write( + join(cli, 'src', 'lazy.ts'), + "import { greet } from '../../../packages/core/dist/index.js';\n" + + "export const later = (): string => greet('lazy');\n", + ); + write( + join(cli, 'src', 'facade.ts'), + "export { greet } from '../../../packages/core/dist/index.js';\n", + ); + const built = await build({ + absWorkingDir: cli, + entryPoints: { index: 'src/index.ts' }, + outdir: 'dist', + format: 'esm', + platform: 'node', + target: 'node22', + bundle: true, + splitting: true, + minify: true, + metafile: true, + write: false, + sourcemap: 'external', + sourcesContent, + logLevel: 'silent', + }); + const emitted = {}; + for (const output of built.outputFiles) { + const name = basename(output.path); + if (name.endsWith('.map')) { + write(join(cli, 'dist', name), JSON.stringify(JSON.parse(output.text))); + } else { + emitted[name] = output.text; + write( + join(cli, 'dist', name), + output.text + mapTrailer(name), + name === 'index.js' ? 0o755 : 0o644, + ); + } + } + write(join(cli, 'dist', 'metafile-esm.json'), JSON.stringify(built.metafile)); + if (!upstreamMapAtBundle) writeFileSync(upstreamMap, upstreamMapBytes); + const staging = join(scratch, `staging-${count}`); + mkdirSync(staging); + chmodSync(staging, 0o700); + return { root, core, cli, dist: join(cli, 'dist'), staging, metafile: built.metafile, emitted }; +} + +/** Refuses with `code` before writing anything: the package and staging stay exactly as they were. */ +function refusesBeforeWriting(candidate, code) { + const before = snapshot(candidate.root); + assert.throws( + () => finalizeCandidate({ repository: candidate.root, staging: candidate.staging }), + (error) => error instanceof CandidateFinalizationError && error.code === code, + ); + assert.deepEqual(snapshot(candidate.root), before); + assert.deepEqual(readdirSync(candidate.staging), []); +} + +function editJson(path, change) { + const value = JSON.parse(readFileSync(path, 'utf8')); + change(value); + writeFileSync(path, JSON.stringify(value)); +} + +/** Runtime outputs to which no input contributes a byte: code-splitting facades. */ +function facadeNames(candidate) { + return Object.entries(candidate.metafile.outputs) + .filter( + ([key, output]) => + !key.endsWith('.map') && + Object.values(output.inputs).every((contribution) => contribution.bytesInOutput === 0), + ) + .map(([key]) => basename(key)) + .sort(); +} + +test('finalizes a chained candidate into private repository-relative maps and public JS only', async () => { + const candidate = await fixture(); + const original = Object.fromEntries( + readdirSync(candidate.dist) + .filter((name) => name.endsWith('.map')) + .map((name) => [name, JSON.parse(readFileSync(join(candidate.dist, name), 'utf8'))]), + ); + const metafileBytes = readFileSync(join(candidate.dist, 'metafile-esm.json')); + const record = finalizeCandidate({ repository: candidate.root, staging: candidate.staging }); + + const names = Object.keys(candidate.emitted).sort(); + assert.ok(names.length >= 4, 'fixture has an entry, a shared chunk, a lazy chunk and a facade'); + assert.deepEqual(readdirSync(candidate.dist).sort(), names); + for (const name of names) { + // Only the terminal trailer is gone; every executable byte, line and column is unchanged. + assert.equal(readFileSync(join(candidate.dist, name), 'utf8'), candidate.emitted[name]); + } + assert.deepEqual(readdirSync(candidate.staging).sort(), [ + 'finalized.json', + 'maps', + 'metafile-esm.json', + ]); + assert.ok(readFileSync(join(candidate.staging, 'metafile-esm.json')).equals(metafileBytes)); + assert.deepEqual( + readdirSync(join(candidate.staging, 'maps')).sort(), + names.map((name) => `${name}.map`), + ); + const sources = new Set(); + for (const name of names) { + const map = JSON.parse(readFileSync(join(candidate.staging, 'maps', `${name}.map`), 'utf8')); + assert.equal('sourcesContent' in map, false); + assert.equal(map.mappings, original[`${name}.map`].mappings); + assert.deepEqual(map.names, original[`${name}.map`].names); + for (const source of map.sources) sources.add(source); + } + assert.deepEqual([...sources].sort(), [ + 'apps/cli/src/index.ts', + 'apps/cli/src/lazy.ts', + 'packages/core/src/index.ts', + ]); + assert.equal(record.upstreamMaps, 2); + assert.deepEqual( + JSON.parse(readFileSync(join(candidate.staging, 'finalized.json'), 'utf8')), + record, + ); + for (const output of record.outputs) { + const bytes = readFileSync(join(candidate.cli, output.path)); + assert.equal(output.bytes, bytes.length); + } + const facades = facadeNames(candidate); + assert.equal(facades.length, 2, 'both facade kinds are exercised'); + const inputs = (name) => Object.keys(candidate.metafile.outputs[`dist/${name}`].inputs).length; + assert.deepEqual(facades.map(inputs).sort(), [0, 1]); + if (posixModes) { + assert.equal(statSync(candidate.staging).mode & 0o777, 0o700); + assert.equal(statSync(join(candidate.staging, 'maps')).mode & 0o777, 0o700); + for (const name of readdirSync(join(candidate.staging, 'maps'))) { + assert.equal(statSync(join(candidate.staging, 'maps', name)).mode & 0o777, 0o600); + } + for (const name of names) { + assert.equal( + statSync(join(candidate.dist, name)).mode & 0o777, + name === 'index.js' ? 0o755 : 0o644, + ); + } + } +}); + +test('refuses a missing upstream map before the bundler can fall back to compiled JS', async () => { + const candidate = await fixture(); + rmSync(join(candidate.core, 'dist', 'index.js.map')); + refusesBeforeWriting(candidate, 'UPSTREAM_MAP_MISSING'); +}); + +test('refuses a final map that the bundler silently chained to compiled JS', async () => { + // Bundled while the upstream map was absent, then restored: only the final source set shows it. + const candidate = await fixture({ upstreamMapAtBundle: false }); + refusesBeforeWriting(candidate, 'MAP_SOURCE_NOT_ORIGINAL'); +}); + +for (const [label, change, code] of [ + ['embedded sources', (map) => (map.sourcesContent = ['x']), 'UPSTREAM_MAP_SHAPE'], + ['a source root', (map) => (map.sourceRoot = 'src'), 'UPSTREAM_MAP_SHAPE'], + ['another version', (map) => (map.version = 2), 'UPSTREAM_MAP_SHAPE'], + ['another file', (map) => (map.file = 'other.js'), 'UPSTREAM_MAP_SHAPE'], + ['no sources', (map) => (map.sources = []), 'UPSTREAM_MAP_SHAPE'], + ['malformed mappings', (map) => (map.mappings = 'AAAA!'), 'UPSTREAM_MAP_SHAPE'], + [ + 'a URL source', + (map) => (map.sources = ['https://example.invalid/index.ts']), + 'MAP_SOURCE_PATH', + ], + ['an absolute source', (map) => (map.sources = ['/index.ts']), 'MAP_SOURCE_PATH'], + ['a backslash source', (map) => (map.sources = ['..\\src\\index.ts']), 'MAP_SOURCE_PATH'], + [ + 'an escaping source', + (map) => (map.sources = ['../../../../outside.ts']), + 'MAP_SOURCE_NOT_ORIGINAL', + ], + [ + 'another package', + (map) => (map.sources = ['../../shared/src/index.ts']), + 'MAP_SOURCE_NOT_ORIGINAL', + ], + ['a declaration', (map) => (map.sources = ['../src/index.d.ts']), 'MAP_SOURCE_NOT_ORIGINAL'], + ['compiled JS', (map) => (map.sources = ['index.js']), 'MAP_SOURCE_NOT_ORIGINAL'], + [ + 'a JS file under the source root', + (map) => (map.sources = ['../src/index.js']), + 'MAP_SOURCE_NOT_ORIGINAL', + ], + ['a missing source', (map) => (map.sources = ['../src/missing.ts']), 'MAP_SOURCE_MISSING'], +]) { + test(`refuses an upstream map with ${label}`, async () => { + const candidate = await fixture(); + editJson(join(candidate.core, 'dist', 'index.js.map'), change); + refusesBeforeWriting(candidate, code); + }); +} + +test('refuses an unparsable upstream map and an inexact upstream map link', async () => { + const unparsable = await fixture(); + writeFileSync(join(unparsable.core, 'dist', 'index.js.map'), '{'); + refusesBeforeWriting(unparsable, 'MALFORMED_JSON'); + for (const change of [ + (js) => js.slice(0, -mapTrailer('index.js').length), + (js) => `${js}\n`, + (js) => js.replace(mapTrailer('index.js'), mapTrailer('other.js')), + (js) => `// sourceMappingURL in a comment\n${js}`, + ]) { + const candidate = await fixture(); + const path = join(candidate.core, 'dist', 'index.js'); + writeFileSync(path, change(readFileSync(path, 'utf8'))); + refusesBeforeWriting(candidate, 'UPSTREAM_MAP_LINK'); + } +}); + +test('refuses final maps that embed sources, lose coverage or carry unreviewed fields', async () => { + refusesBeforeWriting(await fixture({ sourcesContent: true }), 'EMBEDDED_SOURCES'); + for (const [change, code] of [ + [(map) => map.sources.pop(), 'MAP_SOURCE_COVERAGE'], + [(map) => map.sources.push('../src/lazy.ts'), 'MAP_SOURCE_COVERAGE'], + [(map) => map.sources.push(map.sources[0]), 'MAP_SOURCE_DUPLICATE'], + [(map) => (map.x_google_ignoreList = []), 'MAP_SHAPE'], + [(map) => (map.sections = []), 'MAP_SHAPE'], + [(map) => (map.sourceRoot = 'src'), 'MAP_SHAPE'], + [(map) => (map.file = 'other.js'), 'MAP_SHAPE'], + [(map) => (map.mappings = ''), 'MAP_SHAPE'], + [(map) => (map.version = 2), 'MAP_SHAPE'], + [(map) => (map.sources = ['https://example.invalid/a.ts']), 'MAP_SOURCE_PATH'], + ]) { + const candidate = await fixture(); + editJson(join(candidate.dist, 'index.js.map'), change); + refusesBeforeWriting(candidate, code); + } + const candidate = await fixture(); + editJson(join(candidate.dist, `${facadeNames(candidate)[0]}.map`), (map) => { + map.sources = ['../src/facade.ts']; + map.mappings = 'AAAA'; + }); + refusesBeforeWriting(candidate, 'MAP_SHAPE'); +}); + +test('removes only the exact terminal trailer that tsup appended after esbuild sized the output', async () => { + for (const change of [ + (js) => js.slice(0, -mapTrailer('index.js').length), + (js) => `${js}\n`, + (js) => js.replace(mapTrailer('index.js'), mapTrailer('chunk.js')), + (js) => `${js}${mapTrailer('index.js')}`, + ]) { + const candidate = await fixture(); + const path = join(candidate.dist, 'index.js'); + writeFileSync(path, change(readFileSync(path, 'utf8'))); + refusesBeforeWriting(candidate, 'FINAL_MAP_LINK'); + } + const candidate = await fixture(); + const path = join(candidate.dist, 'index.js'); + writeFileSync( + path, + readFileSync(path, 'utf8').replace(mapTrailer('index.js'), `;${mapTrailer('index.js')}`), + ); + refusesBeforeWriting(candidate, 'TRAILER_BYTES'); +}); + +test('accounts for every emitted file and every consumed input', async () => { + const extra = await fixture(); + writeFileSync(join(extra.dist, 'notes.txt'), 'x'); + refusesBeforeWriting(extra, 'OUTPUT_INVENTORY'); + const directory = await fixture(); + mkdirSync(join(directory.dist, 'nested')); + refusesBeforeWriting(directory, 'OUTPUT_INVENTORY'); + const missing = await fixture(); + rmSync(join(missing.dist, 'index.js.map')); + refusesBeforeWriting(missing, 'OUTPUT_INVENTORY'); + const unpaired = await fixture(); + editJson(join(unpaired.dist, 'metafile-esm.json'), (metafile) => { + delete metafile.outputs['dist/index.js.map']; + }); + refusesBeforeWriting(unpaired, 'MAP_PAIRING'); + const foreign = await fixture(); + editJson(join(foreign.dist, 'metafile-esm.json'), (metafile) => { + metafile.inputs['../../node_modules/left-pad/index.js'] = { + bytes: 1, + imports: [], + format: 'esm', + }; + }); + refusesBeforeWriting(foreign, 'UNSUPPORTED_INPUT'); + if (posixModes) { + const linked = await fixture(); + symlinkSync(join(linked.dist, 'index.js'), join(linked.dist, 'alias.js')); + refusesBeforeWriting(linked, 'OUTPUT_INVENTORY'); + // A link standing in for an expected output has the right name but is not an ordinary file. + const replaced = await fixture(); + const map = join(replaced.dist, 'index.js.map'); + writeFileSync(join(replaced.root, 'elsewhere.map'), readFileSync(map)); + rmSync(map); + symlinkSync(join(replaced.root, 'elsewhere.map'), map); + refusesBeforeWriting(replaced, 'OUTPUT_INVENTORY'); + } +}); + +test('refuses an orphan map entry and validates the upstream map of a zero-byte input', async () => { + const orphan = await fixture(); + editJson(join(orphan.dist, 'metafile-esm.json'), (metafile) => { + metafile.outputs['dist/orphan.js.map'] = { imports: [], exports: [], inputs: {}, bytes: 1 }; + }); + refusesBeforeWriting(orphan, 'MAP_PAIRING'); + // types.js contributes no byte to any output, yet its map is a consumed upstream map. + const zero = await fixture(); + editJson(join(zero.core, 'dist', 'types.js.map'), (map) => (map.sourceRoot = 'src')); + refusesBeforeWriting(zero, 'UPSTREAM_MAP_SHAPE'); +}); + +test('requires an empty private staging directory outside the candidate', async () => { + const candidate = await fixture(); + for (const [staging, code] of [ + ['relative-staging', 'STAGING_PATH'], + [join(scratch, 'absent-staging'), 'STAGING_PATH'], + [join(candidate.root, 'apps'), 'STAGING_PATH'], + [scratch, 'STAGING_PATH'], + ]) { + assert.throws( + () => finalizeCandidate({ repository: candidate.root, staging }), + (error) => error instanceof CandidateFinalizationError && error.code === code, + ); + } + writeFileSync(join(candidate.staging, 'left-over'), 'x'); + refusesBeforeWritingIgnoringStaging(candidate, 'STAGING_NOT_EMPTY'); + rmSync(join(candidate.staging, 'left-over')); + if (posixModes) { + chmodSync(candidate.staging, 0o755); + refusesBeforeWriting(candidate, 'STAGING_NOT_PRIVATE'); + chmodSync(candidate.staging, 0o700); + const link = join(scratch, `staging-link-${count}`); + symlinkSync(candidate.staging, link); + assert.throws( + () => finalizeCandidate({ repository: candidate.root, staging: link }), + (error) => error instanceof CandidateFinalizationError && error.code === 'STAGING_PATH', + ); + } +}); + +function refusesBeforeWritingIgnoringStaging(candidate, code) { + const before = snapshot(candidate.root); + assert.throws( + () => finalizeCandidate({ repository: candidate.root, staging: candidate.staging }), + (error) => error instanceof CandidateFinalizationError && error.code === code, + ); + assert.deepEqual(snapshot(candidate.root), before); +} + +test('re-checks the finite closure and the migration copy on the finalized files', async () => { + if (posixModes) { + const mode = await fixture(); + chmodSync(join(mode.dist, 'index.js'), 0o644); + assert.throws( + () => finalizeCandidate({ repository: mode.root, staging: mode.staging }), + (error) => error instanceof BundleClosureError && error.code === 'OUTPUT_MODE', + ); + } + const migration = await fixture(); + writeFileSync(join(migration.cli, 'drizzle', '0000_init.sql'), 'drifted\n'); + assert.throws( + () => finalizeCandidate({ repository: migration.root, staging: migration.staging }), + (error) => error instanceof CandidateFinalizationError && error.code === 'MIGRATION_COPY', + ); +}); + +test('ordinary package and CLI builds stay map-free outside a named candidate', () => { + const root = new URL('../../', import.meta.url); + for (const name of ['shared', 'llm', 'core', 'db', 'mcp']) { + const path = fileURLToPath(new URL(`packages/${name}/tsconfig.build.json`, root)); + const read = ts.readConfigFile(path, ts.sys.readFile); + assert.equal(read.error, undefined); + const { options } = ts.parseJsonConfigFileContent(read.config, ts.sys, dirname(path)); + assert.equal(options.sourceMap, false, name); + assert.equal(options.declarationMap, false, name); + assert.notEqual(options.inlineSourceMap, true, name); + assert.notEqual(options.inlineSources, true, name); + } + const config = readFileSync(new URL('apps/cli/tsup.config.ts', root), 'utf8'); + assert.match( + config, + /^const CANDIDATE = process\.env\.RELAVIUM_CLI_CANDIDATE_STAGING !== undefined;$/mu, + ); + assert.deepEqual(config.match(/^\s*sourcemap:.*$/gmu), [' sourcemap: CANDIDATE,']); + assert.match(config, /if \(CANDIDATE\) options\.sourcesContent = false;/u); +}); diff --git a/tools/cli-package/pack.mjs b/tools/cli-package/pack.mjs index eec38369..1faa39e0 100644 --- a/tools/cli-package/pack.mjs +++ b/tools/cli-package/pack.mjs @@ -1,40 +1,27 @@ -/** Fresh dependency emit, sole CLI prepack, independent seal. Never calls a cached build. */ +/** + * Command-line entry point of the package producer (`produce.mjs`): one empty artifact directory in, + * one tarball and its public receipt out. Never builds in the developer's workspace and never calls + * a cached build. + */ import assert from 'node:assert/strict'; -import { mkdtempSync, mkdirSync, readdirSync, rmSync, realpathSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join, resolve } from 'node:path'; +import { resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { isolatedEnvironment, runPnpm } from './process.mjs'; -import { sealPackage } from './seal.mjs'; -import { emitFreshPackage } from './pipeline.mjs'; +import { producePackage } from './produce.mjs'; assert.equal(process.argv.length, 3, 'one empty artifact directory required'); -const repository = fileURLToPath(new URL('../../', import.meta.url)); -const destination = resolve(process.argv[2]); -mkdirSync(destination, { recursive: true }); -assert.deepEqual(readdirSync(destination), [], 'PACKED_CLI_FRESH_DESTINATION_REQUIRED'); -const home = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-package-build-'))); -try { - const env = isolatedEnvironment(home); - const version = await runPnpm(['--version'], { cwd: repository, env }); - assert.equal(version.stdout.trim(), '9.12.3', 'PACKED_CLI_PNPM_VERSION'); - const receipt = await emitFreshPackage({ - repository, - destination, - env, - pnpm: runPnpm, - removeDist: (path) => rmSync(path, { recursive: true, force: true }), - seal: sealPackage, - }); - console.log( - JSON.stringify({ - mode: receipt.mode, - source: receipt.source, - sha256: receipt.tarballSha256, - files: receipt.files.length, - mapsAndSymbolication: 'not implemented until Wave 3 Step 9', - }), - ); -} finally { - rmSync(home, { recursive: true, force: true }); -} +const { receipt, identity } = await producePackage({ + repository: fileURLToPath(new URL('../../', import.meta.url)), + destination: resolve(process.argv[2]), + environment: process.env, +}); +console.log( + JSON.stringify({ + mode: receipt.mode, + source: receipt.source, + sha256: receipt.tarballSha256, + files: receipt.files.length, + candidateFiles: identity.files, + privateMaps: 'staged privately, then discarded', + symbolication: 'not implemented until Wave 3 Step 9a.2', + }), +); diff --git a/tools/cli-package/pipeline.mjs b/tools/cli-package/pipeline.mjs index 7317cfd4..8beb5ddb 100644 --- a/tools/cli-package/pipeline.mjs +++ b/tools/cli-package/pipeline.mjs @@ -1,25 +1,55 @@ import assert from 'node:assert/strict'; import { join } from 'node:path'; +import { CANDIDATE_STAGING_ENV, ENGINE_PACKAGES } from './finalize.mjs'; -/** One ordered emitter; ports are tooling boundaries, never shipped product hooks. */ -export async function emitFreshPackage({ repository, destination, env, pnpm, removeDist, seal }) { - for (const name of ['shared', 'llm', 'core', 'db', 'mcp']) { +/** The installed compiler's candidate overrides: external upstream maps, no embedded sources or incremental state. */ +export const CANDIDATE_TSC_ARGS = Object.freeze([ + 'exec', + 'tsc', + '-p', + 'tsconfig.build.json', + '--sourceMap', + 'true', + '--inlineSources', + 'false', + '--incremental', + 'false', +]); + +/** + * One ordered emitter for an owned candidate tree; ports are tooling boundaries, never shipped + * product hooks. The inlined dependencies compile fresh with upstream maps, then prepack performs + * the sole final CLI build and finalizes the candidate into the private `staging` directory. + */ +export async function emitFreshPackage({ + repository, + destination, + staging, + env, + pnpm, + removeBuildState, + seal, +}) { + for (const name of ENGINE_PACKAGES) { const cwd = join(repository, 'packages', name); - await removeDist(join(cwd, 'dist')); - await pnpm(['exec', 'tsc', '-p', 'tsconfig.build.json', '--incremental', 'false'], { - cwd, - env, - }); + await removeBuildState(cwd); + await pnpm([...CANDIDATE_TSC_ARGS], { cwd, env }); } const packed = await pnpm(['pack', '--pack-destination', destination], { cwd: join(repository, 'apps/cli'), - env, + env: { ...env, [CANDIDATE_STAGING_ENV]: staging }, }); - // Independently observe the installed packer's sole prepack build, rather than merely counting API calls. + // Independently observe the installed packer's sole prepack build and its one finalization, + // rather than merely counting API calls. assert.equal( (packed.stdout.match(/\bESM Build start\b/gu) ?? []).length, 1, 'PACKED_CLI_SOLE_PREPACK_BUILD_REQUIRED', ); + assert.equal( + (packed.stdout.match(/^✓ CLI candidate finalized: /gmu) ?? []).length, + 1, + 'PACKED_CLI_CANDIDATE_FINALIZATION_REQUIRED', + ); return seal(destination); } diff --git a/tools/cli-package/pipeline.test.mjs b/tools/cli-package/pipeline.test.mjs index cbf9a989..d54a0d5e 100644 --- a/tools/cli-package/pipeline.test.mjs +++ b/tools/cli-package/pipeline.test.mjs @@ -2,23 +2,28 @@ import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import { test } from 'node:test'; -import { emitFreshPackage } from './pipeline.mjs'; +import { CANDIDATE_TSC_ARGS, emitFreshPackage } from './pipeline.mjs'; -function fixture(failAt, stdout = 'ESM Build start\nESM Build success') { - const repository = join(process.cwd(), 'synthetic-owned-checkout'); +const BUILT = 'ESM Build start\nESM Build success\n'; +const FINALIZED = '✓ CLI candidate finalized: 3 output(s), 3 private map(s), 2 upstream map(s)\n'; + +function fixture(failAt, stdout = `${BUILT}${FINALIZED}`) { + const repository = join(process.cwd(), 'synthetic-owned-candidate'); const destination = join(process.cwd(), 'synthetic-public-output'); + const staging = join(process.cwd(), 'synthetic-private-staging'); const calls = []; let commands = 0; return { calls, repository, destination, + staging, env: { HOME: 'synthetic' }, - async removeDist(path) { + async removeBuildState(path) { calls.push({ remove: path }); }, async pnpm(args, options) { - calls.push({ args, cwd: options.cwd }); + calls.push({ args, cwd: options.cwd, env: options.env }); commands += 1; if (commands === failAt) throw new Error('synthetic dependency or prepack failure'); return { stdout }; @@ -30,18 +35,20 @@ function fixture(failAt, stdout = 'ESM Build start\nESM Build success') { }; } -test('fresh pipeline orders five dependency emits before the sole prepack and independent seal', async () => { +test('fresh pipeline orders five mapped dependency emits before the sole candidate prepack and seal', async () => { const ports = fixture(); assert.equal(await emitFreshPackage(ports), 'actual inspected receipt'); const commands = ports.calls.filter((call) => call.args); + const packages = ['shared', 'llm', 'core', 'db', 'mcp'].map((name) => + join(ports.repository, 'packages', name), + ); assert.deepEqual( commands.map((call) => call.cwd), - [ - ...['shared', 'llm', 'core', 'db', 'mcp'].map((name) => - join(ports.repository, 'packages', name), - ), - join(ports.repository, 'apps/cli'), - ], + [...packages, join(ports.repository, 'apps/cli')], + ); + assert.deepEqual( + ports.calls.filter((call) => call.remove).map((call) => call.remove), + packages, ); assert.deepEqual( commands.slice(0, 5).map((call) => call.args), @@ -50,13 +57,23 @@ test('fresh pipeline orders five dependency emits before the sole prepack and in 'tsc', '-p', 'tsconfig.build.json', + '--sourceMap', + 'true', + '--inlineSources', + 'false', '--incremental', 'false', ]), ); + assert.deepEqual(CANDIDATE_TSC_ARGS, commands[0].args); + // Only the sole prepack learns the private staging directory. + assert.ok(commands.slice(0, 5).every((call) => call.env === ports.env)); assert.deepEqual(commands[5].args, ['pack', '--pack-destination', ports.destination]); + assert.deepEqual(commands[5].env, { + HOME: 'synthetic', + RELAVIUM_CLI_CANDIDATE_STAGING: ports.staging, + }); assert.deepEqual(ports.calls.at(-1), { seal: ports.destination }); - assert.equal(ports.calls.filter((call) => call.remove).length, 5); assert.ok(commands.every((call) => !call.args.includes('turbo') && !call.args.includes('build'))); }); @@ -71,7 +88,7 @@ for (const failAt of [1, 2, 3, 4, 5, 6]) ); }); -for (const stdout of ['', 'ESM Build start\nESM Build start\n']) +for (const stdout of ['', `${BUILT}${BUILT}${FINALIZED}`, FINALIZED]) test(`missing/duplicate observed prepack build refuses before seal: ${JSON.stringify(stdout)}`, async () => { const ports = fixture(undefined, stdout); await assert.rejects(emitFreshPackage(ports), /SOLE_PREPACK_BUILD_REQUIRED/); @@ -81,9 +98,20 @@ for (const stdout of ['', 'ESM Build start\nESM Build start\n']) ); }); -test('the shipped prepack lifecycle has exactly one build and runs its closure gate', () => { +for (const stdout of [BUILT, `${BUILT}${FINALIZED}${FINALIZED}`, `${BUILT} ${FINALIZED}`]) + test(`missing/duplicate candidate finalization refuses before seal: ${JSON.stringify(stdout)}`, async () => { + const ports = fixture(undefined, stdout); + await assert.rejects(emitFreshPackage(ports), /CANDIDATE_FINALIZATION_REQUIRED/); + assert.equal( + ports.calls.some((call) => call.seal), + false, + ); + }); + +test('the shipped prepack lifecycle is the sole orchestrated build and closure gate', () => { const metadata = JSON.parse( readFileSync(new URL('../../apps/cli/package.json', import.meta.url), 'utf8'), ); - assert.equal(metadata.scripts.prepack, 'tsup && pnpm -w run lint:bundle-closure'); + assert.equal(metadata.scripts.prepack, 'node ../../tools/cli-package/prepack.mjs'); + assert.equal(metadata.scripts.build, 'tsup'); }); diff --git a/tools/cli-package/prepack.mjs b/tools/cli-package/prepack.mjs new file mode 100644 index 00000000..a297b266 --- /dev/null +++ b/tools/cli-package/prepack.mjs @@ -0,0 +1,62 @@ +/** + * The CLI's sole prepack lifecycle (ADR-0111). It runs exactly one installed tsup build, then + * either the ordinary map-free closure guard or — only when the trusted package producer names a + * private staging directory — the candidate finalizer. pnpm packs only after this exits 0, and + * nothing builds, transforms or writes the JS after it. + */ +import { AssertionError } from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { BundleClosureError } from '../bundle-closure/closure.mjs'; +import { + CANDIDATE_STAGING_ENV, + CandidateFinalizationError, + assertPrivateStaging, + finalizeCandidate, +} from './finalize.mjs'; + +const repository = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const cli = join(repository, 'apps', 'cli'); + +/** @param {string} message @returns {never} */ +function fail(message) { + console.error(`✗ ${message}`); + process.exit(1); +} + +/** @param {string} script @param {string} cwd */ +function runNode(script, cwd) { + const result = spawnSync(process.execPath, [script], { cwd, stdio: 'inherit', shell: false }); + return result.error === undefined && result.status === 0; +} + +const staging = process.env[CANDIDATE_STAGING_ENV]; +try { + if (resolve(process.cwd()) !== cli) fail('CLI prepack must run in its package directory'); + // Refuse an unusable staging directory before anything is built. + if (staging !== undefined) assertPrivateStaging(staging, repository); + const tsupManifest = createRequire(join(cli, 'package.json')).resolve('tsup/package.json'); + const tsup = join(dirname(tsupManifest), JSON.parse(readFileSync(tsupManifest, 'utf8')).bin.tsup); + if (!runNode(tsup, cli)) fail('CLI build failed'); + if (staging === undefined) { + if (!runNode(join(repository, 'tools', 'bundle-closure', 'check.mjs'), repository)) { + process.exit(1); + } + } else { + const record = finalizeCandidate({ repository, staging }); + console.log( + `✓ CLI candidate finalized: ${record.outputs.length} output(s), ` + + `${record.maps.length} private map(s), ${record.upstreamMaps} upstream map(s)`, + ); + } +} catch (error) { + // Every known refusal carries a fixed code and at most a repository-relative path. + if (error instanceof CandidateFinalizationError) fail(error.message); + if (error instanceof BundleClosureError) fail(`CLI candidate closure: ${error.message}`); + if (error instanceof AssertionError) fail(`CLI candidate finalization: ${error.message}`); + const code = error instanceof Error && 'code' in error ? String(error.code) : 'UNEXPECTED'; + fail(`CLI candidate finalization failed: ${code}`); +} diff --git a/tools/cli-package/prepack.test.mjs b/tools/cli-package/prepack.test.mjs new file mode 100644 index 00000000..0bc3eabe --- /dev/null +++ b/tools/cli-package/prepack.test.mjs @@ -0,0 +1,133 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { + chmodSync, + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { after, test } from 'node:test'; +import { fileURLToPath } from 'node:url'; + +// The orchestrator runs from a copied tooling tree with a fake tsup that only records that it ran, +// so no case can rebuild the real CLI even if the orchestration regresses. +const tooling = fileURLToPath(new URL('../', import.meta.url)); +const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-prepack-orchestrator-'))); +after(() => rmSync(scratch, { recursive: true, force: true })); +let count = 0; + +function fixture({ buildExit = 0 } = {}) { + count += 1; + const root = join(scratch, `repository-${count}`); + for (const path of [ + 'cli-package/prepack.mjs', + 'cli-package/finalize.mjs', + 'cli-package/files.mjs', + 'bundle-closure/closure.mjs', + 'bundle-closure/inventory.mjs', + 'bundle-closure/check.mjs', + ]) { + mkdirSync(dirname(join(root, 'tools', path)), { recursive: true }); + copyFileSync(join(tooling, path), join(root, 'tools', path)); + } + symlinkSync( + fileURLToPath(new URL('../../node_modules', import.meta.url)), + join(root, 'node_modules'), + process.platform === 'win32' ? 'junction' : 'dir', + ); + const cli = join(root, 'apps', 'cli'); + const tsup = join(cli, 'node_modules', 'tsup'); + mkdirSync(join(tsup, 'dist'), { recursive: true }); + writeFileSync(join(cli, 'package.json'), JSON.stringify({ name: 'relavium', dependencies: {} })); + writeFileSync( + join(tsup, 'package.json'), + JSON.stringify({ name: 'tsup', type: 'module', bin: { tsup: 'dist/cli.js' } }), + ); + const marker = join(root, 'build-ran'); + writeFileSync( + join(tsup, 'dist', 'cli.js'), + `import { appendFileSync } from 'node:fs';\n` + + `appendFileSync(${JSON.stringify(marker)}, (process.env.RELAVIUM_CLI_CANDIDATE_STAGING ?? 'ordinary') + '\\n');\n` + + `console.log('ESM Build start');\nprocess.exit(${buildExit});\n`, + ); + return { root, cli, marker }; +} + +function prepack({ cli, root }, { cwd = cli, staging } = {}) { + const env = { ...process.env }; + delete env.RELAVIUM_CLI_CANDIDATE_STAGING; + if (staging !== undefined) env.RELAVIUM_CLI_CANDIDATE_STAGING = staging; + const result = spawnSync(process.execPath, [join(root, 'tools', 'cli-package', 'prepack.mjs')], { + cwd, + env, + encoding: 'utf8', + timeout: 20_000, + }); + assert.equal(result.error, undefined); + return result; +} + +function privateStaging() { + count += 1; + const staging = join(scratch, `staging-${count}`); + mkdirSync(staging); + chmodSync(staging, 0o700); + return staging; +} + +test('prepack runs only in the CLI package directory', () => { + const candidate = fixture(); + const result = prepack(candidate, { cwd: candidate.root }); + assert.equal(result.status, 1); + assert.match(result.stderr, /✗ CLI prepack must run in its package directory/u); + assert.equal(existsSync(candidate.marker), false); +}); + +test('an unusable staging directory refuses before anything is built', () => { + for (const staging of ['relative', join(scratch, 'absent')]) { + const candidate = fixture(); + const result = prepack(candidate, { staging }); + assert.equal(result.status, 1); + assert.match(result.stderr, /CLI candidate finalization refused: STAGING_PATH/u); + assert.equal(existsSync(candidate.marker), false); + } +}); + +test('a failed build stops before the closure gate or finalization', () => { + const candidate = fixture({ buildExit: 3 }); + const staging = privateStaging(); + const result = prepack(candidate, { staging }); + assert.equal(result.status, 1); + assert.match(result.stderr, /✗ CLI build failed/u); + assert.equal(readFileSync(candidate.marker, 'utf8'), `${staging}\n`); + assert.equal(existsSync(join(staging, 'maps')), false); +}); + +test('one build runs, then the ordinary closure gate or the candidate finalizer judges its output', () => { + // The fake build emits nothing, so both judges must refuse the missing output after one build. + const ordinary = fixture(); + const plain = prepack(ordinary); + assert.equal(plain.status, 1); + assert.equal(readFileSync(ordinary.marker, 'utf8'), 'ordinary\n'); + assert.match(plain.stdout, /ESM Build start/u); + assert.match(plain.stderr, /✗ CLI bundle closure: cannot read valid build inventory/u); + + const candidate = fixture(); + const staging = privateStaging(); + const mapped = prepack(candidate, { staging }); + assert.equal(mapped.status, 1); + assert.equal(readFileSync(candidate.marker, 'utf8'), `${staging}\n`); + assert.match( + mapped.stderr, + /✗ CLI candidate finalization refused: UNREADABLE_FILE \(metafile\)/u, + ); + assert.doesNotMatch(mapped.stdout, /CLI candidate finalized/u); +}); diff --git a/tools/cli-package/produce.mjs b/tools/cli-package/produce.mjs new file mode 100644 index 00000000..e53c05c4 --- /dev/null +++ b/tools/cli-package/produce.mjs @@ -0,0 +1,98 @@ +/** + * The package producer (ADR-0111, Wave 3 Step 9a.1): owned candidate copy, fresh mapped dependency + * emit, sole CLI prepack with candidate finalization, independent seal. It never builds in the + * developer's workspace and never calls a cached build. The private maps, metafile and hash record + * are staged owner-only outside the artifact directory and discarded on every path; only the tarball + * and its public receipt are published. `pack.mjs` is its command-line entry point; the ports are + * tooling seams for tests, never shipped product hooks. + */ +import assert from 'node:assert/strict'; +import { chmodSync, mkdirSync, mkdtempSync, readdirSync, realpathSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { isAbsolute, join, relative } from 'node:path'; +import { assertNoBuildCache, globalExcludesFile, prepareCandidate } from './candidate.mjs'; +import { isolatedEnvironment, runPnpm } from './process.mjs'; +import { sealPackage } from './seal.mjs'; +import { emitFreshPackage } from './pipeline.mjs'; + +/** @param {string} parent @param {string} child */ +function within(parent, child) { + const path = relative(parent, child); + return path === '' || (!path.startsWith('..') && !isAbsolute(path)); +} + +/** + * Produce one package-test artifact from the original checkout `repository` into the empty + * `destination`. The candidate tree and its private staging are siblings in one owner-only temporary + * root that shares no path with the artifact directory or the checkout, and that root is removed + * whether production succeeds or fails; no secure erasure is claimed. + * @param {{repository: string, destination: string, environment: Record, + * temporary?: string, ports?: {prepare?: typeof prepareCandidate, emit?: typeof emitFreshPackage, + * seal?: typeof sealPackage, noBuildCache?: typeof assertNoBuildCache, + * excludesFile?: typeof globalExcludesFile, pnpm?: typeof runPnpm}}} options + */ +export async function producePackage({ + repository, + destination, + environment, + temporary = tmpdir(), + ports = {}, +}) { + const { + prepare = prepareCandidate, + emit = emitFreshPackage, + seal = sealPackage, + noBuildCache = assertNoBuildCache, + excludesFile = globalExcludesFile, + pnpm = runPnpm, + } = ports; + mkdirSync(destination, { recursive: true }); + assert.deepEqual(readdirSync(destination), [], 'PACKED_CLI_FRESH_DESTINATION_REQUIRED'); + const home = realpathSync(mkdtempSync(join(temporary, 'relavium-package-build-'))); + let root; + try { + root = realpathSync(mkdtempSync(join(temporary, 'relavium-package-candidate-'))); + chmodSync(root, 0o700); + const candidate = join(root, 'source'); + const staging = join(root, 'staging'); + mkdirSync(candidate, { mode: 0o700 }); + mkdirSync(staging, { mode: 0o700 }); + chmodSync(staging, 0o700); + const published = realpathSync(destination); + assert.ok( + !within(root, published) && !within(published, root) && !within(repository, root), + 'PACKED_CLI_PRIVATE_STAGING_PLACEMENT', + ); + const env = isolatedEnvironment(home); + const version = await pnpm(['--version'], { cwd: repository, env }); + assert.equal(version.stdout.trim(), '9.12.3', 'PACKED_CLI_PNPM_VERSION'); + + const excludes = await excludesFile({ repository, environment }); + const identity = await prepare({ repository, candidate, environment, home, env, excludes }); + const receipt = await emit({ + repository: candidate, + destination, + staging, + env, + pnpm, + removeBuildState: (cwd) => { + rmSync(join(cwd, 'dist'), { recursive: true, force: true }); + rmSync(join(cwd, 'tsconfig.build.tsbuildinfo'), { force: true }); + }, + seal: (artifactDirectory) => { + noBuildCache(candidate); + return seal(artifactDirectory, { + repository: candidate, + staging, + source: identity.source, + dirty: identity.dirty, + }); + }, + }); + return { receipt, identity }; + } finally { + // Private maps never outlive the producer. + if (root !== undefined) rmSync(root, { recursive: true, force: true }); + rmSync(home, { recursive: true, force: true }); + } +} diff --git a/tools/cli-package/produce.test.mjs b/tools/cli-package/produce.test.mjs new file mode 100644 index 00000000..60545832 --- /dev/null +++ b/tools/cli-package/produce.test.mjs @@ -0,0 +1,179 @@ +import assert from 'node:assert/strict'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + realpathSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { after, test } from 'node:test'; +import { assertNoBuildCache } from './candidate.mjs'; +import { producePackage } from './produce.mjs'; + +const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-produce-'))); +after(() => rmSync(scratch, { recursive: true, force: true })); +const posixModes = process.platform !== 'win32'; +let count = 0; + +function fresh(name) { + count += 1; + const path = join(scratch, `${name}-${count}`); + mkdirSync(path); + return path; +} + +const SHA = 'c'.repeat(40); + +/** + * Fake ports that record what the producer composes. The fake emit writes into the candidate it is + * given, runs the producer's seal wrapper and can plant a build-cache marker first. + */ +function ports({ plantCache = false, failEmit = false } = {}) { + const seen = {}; + return { + seen, + ports: { + pnpm: async (args) => { + assert.deepEqual(args, ['--version']); + return { stdout: '9.12.3\n', stderr: '' }; + }, + excludesFile: async ({ repository }) => { + seen.excludesRepository = repository; + return '/developer/excludes'; + }, + prepare: async (options) => { + seen.prepare = options; + return { source: SHA, dirty: false, files: 7 }; + }, + emit: async (options) => { + seen.emit = options; + seen.stagingMode = statSync(options.staging).mode & 0o777; + seen.rootMode = statSync(join(options.staging, '..')).mode & 0o777; + mkdirSync(join(options.repository, 'apps', 'cli', 'dist'), { recursive: true }); + if (plantCache) writeFileSync(join(options.repository, '.turbo'), 'cache\n'); + if (failEmit) throw new Error('synthetic emit failure'); + writeFileSync(join(options.destination, 'relavium-0.1.1.tgz'), 'tarball'); + return options.seal(options.destination); + }, + noBuildCache: (candidate) => { + seen.noBuildCache = candidate; + assertNoBuildCache(candidate); + }, + seal: async (artifactDirectory, candidate) => { + seen.seal = { artifactDirectory, ...candidate }; + return { mode: 'compiled_package_test', source: candidate.source, files: [] }; + }, + }, + }; +} + +function leftovers(temporary) { + return readdirSync(temporary).filter((name) => name.startsWith('relavium-package-')); +} + +test('the producer builds and seals only the owned candidate, never the original checkout', async () => { + const repository = fresh('original-checkout'); + const destination = join(fresh('artifact-parent'), 'artifact'); + const temporary = fresh('tmp'); + const { seen, ports: fakes } = ports(); + const result = await producePackage({ + repository, + destination, + environment: { HOME: '/home/someone' }, + temporary, + ports: fakes, + }); + assert.deepEqual(result.identity, { source: SHA, dirty: false, files: 7 }); + const candidate = seen.prepare.candidate; + // The candidate and its staging are siblings in one private root outside the checkout and the artifact. + assert.notEqual(candidate, repository); + assert.equal(seen.prepare.repository, repository); + assert.equal(seen.prepare.excludes, '/developer/excludes'); + assert.equal(seen.excludesRepository, repository); + assert.equal(seen.emit.repository, candidate); + assert.equal(seen.emit.destination, destination); + assert.equal(seen.emit.staging, join(candidate, '..', 'staging')); + assert.equal(seen.noBuildCache, candidate); + assert.deepEqual(seen.seal, { + artifactDirectory: destination, + repository: candidate, + staging: seen.emit.staging, + source: SHA, + dirty: false, + }); + if (posixModes) { + assert.equal(seen.stagingMode, 0o700); + assert.equal(seen.rootMode, 0o700); + } + // The private root and the build home are gone, and only the tarball remains in the artifact. + assert.deepEqual(leftovers(temporary), []); + assert.equal(existsSync(candidate), false); + assert.deepEqual(readdirSync(destination), ['relavium-0.1.1.tgz']); +}); + +test('a build cache left in the candidate refuses before the seal, and nothing private remains', async () => { + const temporary = fresh('tmp'); + const { seen, ports: fakes } = ports({ plantCache: true }); + await assert.rejects( + producePackage({ + repository: fresh('original-checkout'), + destination: join(fresh('artifact-parent'), 'artifact'), + environment: {}, + temporary, + ports: fakes, + }), + /CANDIDATE_BUILD_CACHE/u, + ); + assert.equal(seen.seal, undefined); + assert.deepEqual(leftovers(temporary), []); +}); + +test('a failure inside the build still removes the private root', async () => { + const temporary = fresh('tmp'); + const { ports: fakes } = ports({ failEmit: true }); + await assert.rejects( + producePackage({ + repository: fresh('original-checkout'), + destination: join(fresh('artifact-parent'), 'artifact'), + environment: {}, + temporary, + ports: fakes, + }), + /synthetic emit failure/u, + ); + assert.deepEqual(leftovers(temporary), []); +}); + +test('private staging never shares a path with the checkout or the artifact', async () => { + // A temporary directory inside the original checkout would put the private root there. + const repository = fresh('original-checkout'); + const inside = join(repository, 'tmp'); + mkdirSync(inside); + const { seen, ports: fakes } = ports(); + await assert.rejects( + producePackage({ + repository, + destination: join(fresh('artifact-parent'), 'artifact'), + environment: {}, + temporary: inside, + ports: fakes, + }), + /PRIVATE_STAGING_PLACEMENT/u, + ); + assert.equal(seen.prepare, undefined); + assert.deepEqual(leftovers(inside), []); + // A non-empty artifact directory refuses before anything private exists. + const temporary = fresh('tmp'); + const destination = fresh('artifact'); + writeFileSync(join(destination, 'stale.tgz'), 'old'); + await assert.rejects( + producePackage({ repository, destination, environment: {}, temporary, ports: fakes }), + /FRESH_DESTINATION_REQUIRED/u, + ); + assert.deepEqual(leftovers(temporary), []); +}); diff --git a/tools/cli-package/release-guard-core.mjs b/tools/cli-package/release-guard-core.mjs new file mode 100644 index 00000000..5a81890e --- /dev/null +++ b/tools/cli-package/release-guard-core.mjs @@ -0,0 +1,45 @@ +/** + * Interim fail-closed publication guard (Wave 3 Step 9a.1, the Interim release guard ruling of 2026-10-19). + * + * npm publication requires a release candidate: the exact tarball, its original ciphertext and a + * release binding receipt verified against the release custody contract + * (docs/reference/cli/packaging.md). Wave 3 Step 9d implements that custody graph and Wave 4b + * supplies the production recipient; until both exist no artifact can be a release candidate, so + * this guard refuses every artifact. A package-test artifact is never promoted to a release. + * + * It depends on Node built-ins only (the publish job checks out nothing else) and reads directory + * entry names only — never artifact contents. `release-guard.mjs` is the publish job's entry point. + */ +import { readdirSync } from 'node:fs'; + +export class ReleaseGuardError extends Error { + /** @param {string} code */ + constructor(code) { + super(`CLI publication refused: ${code}`); + this.name = 'ReleaseGuardError'; + this.code = code; + } +} + +/** + * @param {string} directory the downloaded artifact directory + * @returns {never} until Step 9d's release custody validator replaces this interim guard + * @throws {ReleaseGuardError} for every artifact + */ +export function requireReleaseCandidate(directory) { + let names; + try { + names = readdirSync(directory); + } catch { + throw new ReleaseGuardError('ARTIFACT_UNREADABLE'); + } + if (!names.includes('release-binding-receipt.json')) { + throw new ReleaseGuardError( + names.includes('package-test-receipt.json') + ? 'PACKAGE_TEST_ARTIFACT' + : 'RELEASE_CANDIDATE_REQUIRED', + ); + } + // No release binding receipt is trusted before the Step 9d custody validator exists. + throw new ReleaseGuardError('RELEASE_CUSTODY_UNAVAILABLE'); +} diff --git a/tools/cli-package/release-guard.mjs b/tools/cli-package/release-guard.mjs new file mode 100644 index 00000000..9fc2be77 --- /dev/null +++ b/tools/cli-package/release-guard.mjs @@ -0,0 +1,26 @@ +/** + * The release workflow's publish-job entry point for the interim publication guard + * (`release-guard-core.mjs`; Wave 3 Step 9a.1, the Interim release guard ruling of 2026-10-19). + * + * It refuses before it evaluates anything: the exit code is 1 from the first statement, so a module + * that fails to load, an unexpected throw or a guard that returned would still refuse, and nothing + * about how the script is invoked (a symlinked or relative path) can skip the refusal. Until Step 9d's + * custody validator replaces it, no artifact passes. It prints one fixed line and never artifact + * contents. + */ +import { resolve } from 'node:path'; + +process.exitCode = 1; +let code = 'GUARD_FAILURE'; +try { + const { ReleaseGuardError, requireReleaseCandidate } = await import('./release-guard-core.mjs'); + try { + if (process.argv.length !== 3) throw new ReleaseGuardError('USAGE'); + requireReleaseCandidate(resolve(process.argv[2])); + } catch (error) { + if (error instanceof ReleaseGuardError) code = error.code; + } +} catch { + // A guard that cannot load its rule still refuses. +} +console.error(`::error::CLI publication refused: ${code}`); diff --git a/tools/cli-package/release-guard.test.mjs b/tools/cli-package/release-guard.test.mjs new file mode 100644 index 00000000..7e8bec54 --- /dev/null +++ b/tools/cli-package/release-guard.test.mjs @@ -0,0 +1,111 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { + copyFileSync, + mkdirSync, + mkdtempSync, + realpathSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, relative } from 'node:path'; +import { after, test } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { ReleaseGuardError, requireReleaseCandidate } from './release-guard-core.mjs'; + +const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-release-guard-'))); +after(() => rmSync(scratch, { recursive: true, force: true })); +const guard = fileURLToPath(new URL('./release-guard.mjs', import.meta.url)); +let count = 0; + +function artifact(files) { + count += 1; + const directory = join(scratch, `artifact-${count}`); + mkdirSync(directory); + for (const [name, contents] of Object.entries(files)) + writeFileSync(join(directory, name), contents); + return directory; +} + +/** Run the guard entry as the publish job does, from `cwd` with the given script path and arguments. */ +function runGuard(script, args, cwd = scratch) { + return spawnSync(process.execPath, [script, ...args], { cwd, encoding: 'utf8', timeout: 10_000 }); +} + +const PACKAGE_TEST = { + 'relavium-0.1.1.tgz': 'tarball', + 'package-test-receipt.json': JSON.stringify({ mode: 'compiled_package_test' }), +}; + +for (const [label, files, code] of [ + ['a package-test artifact', PACKAGE_TEST, 'PACKAGE_TEST_ARTIFACT'], + [ + 'a package-test artifact whose receipt claims release mode', + { ...PACKAGE_TEST, 'package-test-receipt.json': JSON.stringify({ mode: 'release' }) }, + 'PACKAGE_TEST_ARTIFACT', + ], + ['a bare tarball', { 'relavium-0.1.1.tgz': 'tarball' }, 'RELEASE_CANDIDATE_REQUIRED'], + ['an empty artifact', {}, 'RELEASE_CANDIDATE_REQUIRED'], + [ + 'a self-declared release candidate', + { + 'relavium-0.1.1.tgz': 'tarball', + 'relavium-0.1.1.tgz.gpg': 'ciphertext', + 'release-binding-receipt.json': JSON.stringify({ mode: 'release' }), + }, + 'RELEASE_CUSTODY_UNAVAILABLE', + ], +]) { + test(`the interim publication guard refuses ${label}`, () => { + const directory = artifact(files); + assert.throws( + () => requireReleaseCandidate(directory), + (error) => error instanceof ReleaseGuardError && error.code === code, + ); + const run = runGuard(guard, [directory]); + assert.equal(run.status, 1); + assert.equal(run.stdout, ''); + assert.equal(run.stderr, `::error::CLI publication refused: ${code}\n`); + }); +} + +test('the guard refuses an unreadable artifact and wrong usage with one fixed line', () => { + const missing = join(scratch, 'absent'); + assert.throws( + () => requireReleaseCandidate(missing), + (error) => error instanceof ReleaseGuardError && error.code === 'ARTIFACT_UNREADABLE', + ); + for (const args of [[], [scratch, scratch]]) { + const run = runGuard(guard, args); + assert.equal(run.status, 1); + assert.equal(run.stderr, '::error::CLI publication refused: USAGE\n'); + } +}); + +test('the guard refuses however its entry point is reached', () => { + const directory = artifact(PACKAGE_TEST); + const expected = '::error::CLI publication refused: PACKAGE_TEST_ARTIFACT\n'; + // A symlinked checkout directory makes the entry's argv path differ from its loaded real path. + const linked = join(scratch, 'linked-tools'); + symlinkSync(dirname(guard), linked, process.platform === 'win32' ? 'junction' : 'dir'); + // A relative, non-canonical path reaches the same file another way. + const relativePath = relative(scratch, join(linked, '..', 'linked-tools', 'release-guard.mjs')); + for (const script of [join(linked, 'release-guard.mjs'), relativePath]) { + const run = runGuard(script, [directory]); + assert.equal(run.status, 1, script); + assert.equal(run.stderr, expected, script); + } +}); + +test('the guard refuses when its rule cannot be loaded', () => { + // The entry point alone, without the core module beside it, still exits 1 with a fixed line. + const lonely = join(scratch, 'lonely'); + mkdirSync(lonely); + copyFileSync(guard, join(lonely, 'release-guard.mjs')); + const run = runGuard(join(lonely, 'release-guard.mjs'), [artifact(PACKAGE_TEST)]); + assert.equal(run.status, 1); + assert.equal(run.stdout, ''); + assert.equal(run.stderr, '::error::CLI publication refused: GUARD_FAILURE\n'); +}); diff --git a/tools/cli-package/seal.mjs b/tools/cli-package/seal.mjs index 1af581b2..2829b3d0 100644 --- a/tools/cli-package/seal.mjs +++ b/tools/cli-package/seal.mjs @@ -1,17 +1,18 @@ -/** Bind the sole prepack result to independently inspected bytes. No build or archive extraction. */ +/** + * Bind the sole prepack result to independently inspected bytes. No build or archive extraction. + * Every input is explicit: the owned candidate root, its private staging directory and the source + * identity the producer captured from the original checkout. Nothing resolves a repository from + * this module's own location, so a candidate copy cannot be sealed against different files. + */ import assert from 'node:assert/strict'; import { createRequire } from 'node:module'; import { readFileSync, readdirSync, writeFileSync } from 'node:fs'; -import { basename, join, resolve } from 'node:path'; -import { fileURLToPath, pathToFileURL } from 'node:url'; +import { basename, join } from 'node:path'; import { inspectPackageArchive } from './archive.mjs'; import { originFromEnvironment, validatePackageReceipt } from './binding.mjs'; -import { flatFiles, oneTarball, readOrdinaryFile } from './files.mjs'; -import { isolatedEnvironment, runCommand } from './process.mjs'; +import { flatFiles, oneTarball, readOrdinaryFile, sha256 } from './files.mjs'; -const repository = fileURLToPath(new URL('../../', import.meta.url)); - -export function producerNodeMajor(root = repository) { +export function producerNodeMajor(root) { const declared = readOrdinaryFile(join(root, '.nvmrc'), 20).toString('utf8').trim(); assert.match(declared, /^[1-9]\d*$/u, 'PACKED_CLI_INVALID_PRODUCER_NODE'); const major = Number(declared); @@ -22,7 +23,7 @@ export function producerNodeMajor(root = repository) { return major; } -export function expectedPackManifest(root = repository) { +export function expectedPackManifest(root) { const require = createRequire(join(root, 'apps/cli/package.json')); const { parse } = require('yaml'); const catalog = parse(readFileSync(join(root, 'pnpm-workspace.yaml'), 'utf8')).catalog; @@ -48,21 +49,38 @@ export function expectedPackManifest(root = repository) { return manifest; } -export function finalizedPublicFiles(root = repository) { +/** + * The finalized public files of a candidate: exactly the runtime outputs named by its staged + * metafile, byte-identical to the finalizer's private hash record, plus canonical migrations and + * the public documents. The finalized dist holds no map and no build metadata. + */ +export function finalizedPublicFiles(root, staging) { const cli = join(root, 'apps/cli'); const files = new Map(); - const graph = JSON.parse(readOrdinaryFile(join(cli, 'dist/metafile-esm.json'), 4 * 1024 * 1024)); - const outputs = Object.keys(graph.outputs).map((path) => path.replaceAll('\\', '/')); + const graph = JSON.parse(readOrdinaryFile(join(staging, 'metafile-esm.json'), 4 * 1024 * 1024)); + const outputs = Object.keys(graph.outputs) + .map((path) => path.replaceAll('\\', '/')) + .filter((path) => !path.endsWith('.map')); assert.ok(outputs.length > 0 && outputs.length <= 256); + const actual = []; for (const path of outputs) { assert.match(path, /^dist\/[A-Za-z0-9_-]+\.js$/u, 'PACKED_CLI_UNREVIEWED_GRAPH_OUTPUT'); - files.set(`package/${path}`, readOrdinaryFile(join(cli, path), 8 * 1024 * 1024)); + const bytes = readOrdinaryFile(join(cli, path), 8 * 1024 * 1024); + files.set(`package/${path}`, bytes); + actual.push({ path, bytes: bytes.length, sha256: sha256(bytes) }); } assert.deepEqual( readdirSync(join(cli, 'dist')).sort(), - [...outputs.map((path) => basename(path)), 'metafile-esm.json'].sort(), + outputs.map((path) => basename(path)).sort(), 'PACKED_CLI_EXTRA_FINAL_OUTPUT', ); + const record = JSON.parse(readOrdinaryFile(join(staging, 'finalized.json'), 1024 * 1024)); + const byPath = (left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0); + assert.deepEqual( + [...record.outputs].sort(byPath), + actual.sort(byPath), + 'PACKED_CLI_FINALIZED_RECORD_DRIFT', + ); const canonical = flatFiles(join(root, 'packages/db/drizzle'), 'package/drizzle'); const copied = flatFiles(join(cli, 'drizzle'), 'package/drizzle'); assert.deepEqual(copied, canonical, 'PACKED_CLI_MIGRATION_COPY_CHANGED'); @@ -73,25 +91,17 @@ export function finalizedPublicFiles(root = repository) { return files; } -export async function sealPackage(artifactDirectory) { +/** + * @param {string} artifactDirectory the public artifact directory holding exactly one tarball + * @param {{repository: string, staging: string, source: string, dirty: boolean}} candidate + */ +export async function sealPackage(artifactDirectory, { repository, staging, source, dirty }) { const tarball = oneTarball(artifactDirectory); - const source = ( - await runCommand('git', ['--no-replace-objects', 'rev-parse', 'HEAD'], { - cwd: repository, - env: isolatedEnvironment(repository), - }) - ).stdout.trim(); - assert.match(source, /^[0-9a-f]{40}$/u); - const dirty = - ( - await runCommand('git', ['status', '--porcelain'], { - cwd: repository, - env: isolatedEnvironment(repository), - }) - ).stdout !== ''; + assert.match(source, /^[0-9a-f]{40}$/u, 'PACKED_CLI_SOURCE_IDENTITY'); + assert.equal(typeof dirty, 'boolean', 'PACKED_CLI_SOURCE_IDENTITY'); const archive = inspectPackageArchive(readOrdinaryFile(tarball, 16 * 1024 * 1024), { - expectedFiles: finalizedPublicFiles(), - expectedPackage: expectedPackManifest(), + expectedFiles: finalizedPublicFiles(repository, staging), + expectedPackage: expectedPackManifest(repository), }); const origin = originFromEnvironment(process.env); const pnpm = JSON.parse(readFileSync(join(repository, 'package.json'), 'utf8')).packageManager; @@ -113,7 +123,7 @@ export async function sealPackage(artifactDirectory) { validatePackageReceipt(receipt, { source, origin, - nodeMajor: producerNodeMajor(), + nodeMajor: producerNodeMajor(repository), allowLocal: origin.kind === 'local', }); writeFileSync( @@ -123,8 +133,3 @@ export async function sealPackage(artifactDirectory) { ); return receipt; } - -if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { - assert.equal(process.argv.length, 3, 'one artifact directory required'); - console.log(JSON.stringify(await sealPackage(resolve(process.argv[2])))); -} diff --git a/tools/cli-package/seal.test.mjs b/tools/cli-package/seal.test.mjs index 822d181f..00c89704 100644 --- a/tools/cli-package/seal.test.mjs +++ b/tools/cli-package/seal.test.mjs @@ -1,21 +1,44 @@ import assert from 'node:assert/strict'; -import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { createHash } from 'node:crypto'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { test } from 'node:test'; import { finalizedPublicFiles, producerNodeMajor } from './seal.mjs'; +/** A finalized candidate: public JS only in dist, its metafile and hash record in private staging. */ function withFinalizedFixture(body) { const root = mkdtempSync(join(tmpdir(), 'relavium-seal-fixture-')); + const staging = join(root, 'staging'); const write = (path, bytes) => { mkdirSync(dirname(join(root, path)), { recursive: true }); writeFileSync(join(root, path), bytes); }; - const graph = (outputs) => write('apps/cli/dist/metafile-esm.json', JSON.stringify({ outputs })); + const graph = (outputs) => write('staging/metafile-esm.json', JSON.stringify({ outputs })); + const record = (paths) => { + const outputs = paths.map((path) => { + const bytes = readFileSync(join(root, 'apps/cli', path)); + return { + path, + bytes: bytes.length, + sha256: createHash('sha256').update(bytes).digest('hex'), + }; + }); + write( + 'staging/finalized.json', + JSON.stringify({ version: 1, outputs, maps: [], upstreamMaps: 0 }), + ); + }; try { - graph({ 'dist/index.js': {}, 'dist/chunk-ABC.js': {} }); + graph({ + 'dist/index.js': {}, + 'dist/index.js.map': {}, + 'dist/chunk-ABC.js': {}, + 'dist/chunk-ABC.js.map': {}, + }); write('apps/cli/dist/index.js', '#!/usr/bin/env node\nimport "./chunk-ABC.js";\n'); write('apps/cli/dist/chunk-ABC.js', 'export {};\n'); + record(['dist/index.js', 'dist/chunk-ABC.js']); for (const path of ['0000_first.sql', 'meta/_journal.json', 'meta/0000_snapshot.json']) { write(`packages/db/drizzle/${path}`, `canonical ${path}\n`); write(`apps/cli/drizzle/${path}`, `canonical ${path}\n`); @@ -23,7 +46,14 @@ function withFinalizedFixture(body) { for (const name of ['package.json', 'README.md', 'CHANGELOG.md']) write(`apps/cli/${name}`, name); write('LICENSE', 'canonical repository license\n'); - return body({ root, write, graph, remove: (path) => rmSync(join(root, path)) }); + return body({ + root, + staging, + write, + graph, + record, + remove: (path) => rmSync(join(root, path)), + }); } finally { rmSync(root, { recursive: true, force: true }); } @@ -42,9 +72,9 @@ test('producer runtime follows the declared toolchain independently of the fixed }); }); -test('finalized seal inventory contains exact graph bytes, canonical migrations and public documents', () => { - withFinalizedFixture(({ root }) => { - const files = finalizedPublicFiles(root); +test('finalized seal inventory contains exact recorded JS bytes, canonical migrations and public documents', () => { + withFinalizedFixture(({ root, staging }) => { + const files = finalizedPublicFiles(root, staging); assert.deepEqual( [...files.keys()].sort(), [ @@ -69,6 +99,7 @@ test('finalized seal inventory contains exact graph bytes, canonical migrations ); assert.equal(files.get('package/LICENSE').toString(), 'canonical repository license\n'); assert.equal(files.has('package/dist/metafile-esm.json'), false); + assert.equal(files.has('package/dist/index.js.map'), false); }); }); @@ -94,7 +125,6 @@ for (const [name, mutate, refusal] of [ undefined, ], ...[ - 'dist/index.js.map', 'dist/nested/index.js', '../outside.js', '/dist/index.js', @@ -112,10 +142,27 @@ for (const [name, mutate, refusal] of [ /PACKED_CLI_EXTRA_FINAL_OUTPUT/, ], [ - 'unlisted emitted map', + 'a map left in the public package', ({ write }) => write('apps/cli/dist/index.js.map', '{}'), /PACKED_CLI_EXTRA_FINAL_OUTPUT/, ], + [ + 'build metadata left in the public package', + ({ write }) => write('apps/cli/dist/metafile-esm.json', '{}'), + /PACKED_CLI_EXTRA_FINAL_OUTPUT/, + ], + [ + 'JS changed after the finalizer recorded it', + ({ write }) => write('apps/cli/dist/chunk-ABC.js', 'export const late = 1;\n'), + /PACKED_CLI_FINALIZED_RECORD_DRIFT/, + ], + [ + 'a hash record missing an output', + ({ record }) => record(['dist/index.js']), + /PACKED_CLI_FINALIZED_RECORD_DRIFT/, + ], + ['a missing hash record', ({ remove }) => remove('staging/finalized.json'), undefined], + ['a missing staged metafile', ({ remove }) => remove('staging/metafile-esm.json'), undefined], [ 'changed migration', ({ write }) => write('apps/cli/drizzle/0000_first.sql', 'changed'), @@ -156,7 +203,7 @@ for (const [name, mutate, refusal] of [ test(`finalized seal refuses ${name}`, () => { withFinalizedFixture((fixture) => { mutate(fixture); - assert.throws(() => finalizedPublicFiles(fixture.root), refusal); + assert.throws(() => finalizedPublicFiles(fixture.root, fixture.staging), refusal); }); }); } @@ -165,10 +212,10 @@ test( 'finalized seal refuses a symlink masquerading as an emitted file', { skip: process.platform === 'win32' }, () => { - withFinalizedFixture(({ root, remove }) => { + withFinalizedFixture(({ root, staging, remove }) => { remove('apps/cli/dist/chunk-ABC.js'); symlinkSync(join(root, 'LICENSE'), join(root, 'apps/cli/dist/chunk-ABC.js')); - assert.throws(() => finalizedPublicFiles(root), /PACKED_CLI_FILE_KIND_OR_BOUND/); + assert.throws(() => finalizedPublicFiles(root, staging), /PACKED_CLI_FILE_KIND_OR_BOUND/); }); }, ); diff --git a/tools/cli-package/smoke/installation.mjs b/tools/cli-package/smoke/installation.mjs new file mode 100644 index 00000000..f4989ce7 --- /dev/null +++ b/tools/cli-package/smoke/installation.mjs @@ -0,0 +1,19 @@ +import { basename, dirname, sep } from 'node:path'; + +/** + * The consumer's node_modules directory that owns an installed package's real path. With pnpm's + * virtual store that path is `/.pnpm//node_modules/`, so the owner is the + * directory holding the store, not the store's per-package node_modules; without a virtual store + * it is the nearest node_modules ancestor. An unrelated node_modules further up never owns it. + * @param {string} packageRoot the installed package's absolute real path + * @returns {string | undefined} + */ +export function installedModulesOwner(packageRoot) { + const store = `${sep}node_modules${sep}.pnpm${sep}`; + const index = packageRoot.lastIndexOf(store); + if (index >= 0) return packageRoot.slice(0, index + `${sep}node_modules`.length); + for (let parent = dirname(packageRoot); parent !== dirname(parent); parent = dirname(parent)) { + if (basename(parent) === 'node_modules') return parent; + } + return undefined; +} diff --git a/tools/cli-package/smoke/installation.test.mjs b/tools/cli-package/smoke/installation.test.mjs new file mode 100644 index 00000000..0766531b --- /dev/null +++ b/tools/cli-package/smoke/installation.test.mjs @@ -0,0 +1,43 @@ +import assert from 'node:assert/strict'; +import { join, parse } from 'node:path'; +import { test } from 'node:test'; +import { installedModulesOwner } from './installation.mjs'; + +const root = parse(process.cwd()).root; +const path = (...parts) => join(root, ...parts); + +test('a pnpm virtual-store installation is owned by the node_modules that holds the store', () => { + const home = path('tmp', 'consumer'); + const installed = join( + home, + 'node_modules', + '.pnpm', + 'relavium@0.1.1', + 'node_modules', + 'relavium', + ); + // The nearest node_modules is the store's per-package directory; dependencies resolve beside it. + assert.equal(installedModulesOwner(installed), join(home, 'node_modules')); +}); + +test('an unrelated node_modules above the installation never owns it', () => { + const home = path('work', 'node_modules', 'scratch', 'consumer'); + assert.equal( + installedModulesOwner( + join(home, 'node_modules', '.pnpm', 'relavium@0.1.1', 'node_modules', 'relavium'), + ), + join(home, 'node_modules'), + ); + assert.equal( + installedModulesOwner(join(home, 'node_modules', 'relavium')), + join(home, 'node_modules'), + ); +}); + +test('a flat installation is owned by its nearest node_modules; none is no owner', () => { + assert.equal( + installedModulesOwner(path('tmp', 'consumer', 'node_modules', 'relavium')), + path('tmp', 'consumer', 'node_modules'), + ); + assert.equal(installedModulesOwner(path('tmp', 'consumer', 'relavium')), undefined); +}); diff --git a/tools/cli-package/smoke/worker.mjs b/tools/cli-package/smoke/worker.mjs index d5712326..d54470c8 100644 --- a/tools/cli-package/smoke/worker.mjs +++ b/tools/cli-package/smoke/worker.mjs @@ -20,6 +20,7 @@ import { assertInstalledMcp, assertInstalledWorkflows } from './workflows.mjs'; import { assertExit, explicitEnvironment, invoke, ndjson } from './process.mjs'; import { assertInstalledNativeProcesses, InstalledNativeControlError } from './native-process.mjs'; import { emitInstalledWorkerFailure } from './failure-diagnostic.mjs'; +import { installedModulesOwner } from './installation.mjs'; const directory = dirname(fileURLToPath(import.meta.url)); const repository = resolve(directory, '../../..'); @@ -60,9 +61,7 @@ async function exerciseInstalledSmoke({ packageRoot, supportRoot, artifactSha256 const require = createRequire(join(packageRoot, 'package.json')); const entry = join(packageRoot, 'dist/index.js'); const before = snapshot(packageRoot); - let installedModules; - for (let parent = dirname(packageRoot); parent !== dirname(parent); parent = dirname(parent)) - if (parent.endsWith(`${sep}node_modules`)) installedModules = parent; + const installedModules = installedModulesOwner(packageRoot); assert.ok(installedModules, 'Installed package must have an explicit node_modules owner'); const root = mkdtempSync(join(supportRoot, 'installed-smoke-')); const log = join(root, 'guard.ndjson'); From 8ddb7ec26023e22ae40cfbfc563524a2a2e53014 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Thu, 22 Oct 2026 06:57:09 +0300 Subject: [PATCH 23/97] fix(repo): address the first Step 9a.1 review round Round 1 (at a72c7ecd) reported nine verified findings, two medium; its plan lens was clean. - An interrupted producer (SIGINT, SIGTERM) left its private maps, metafile, hash record and candidate copy in the temporary directory while its detached pnpm pack ran on. runCommand now tracks every live child tree and stopLiveCommands() kills them synchronously; the producer handles SIGINT, SIGTERM and SIGHUP by stopping them, removing the private root and build home, emptying the artifact directory and exiting with the signal's code. A producer killed outright is a named limit. - Private staging must be canonical and outside the candidate by real path, the repository's included. - Killing tests for the global-excludes wiring (and check.mjs's producer environment, now producerEnvironment()), the placement check's artifact half, the candidate's file-count and byte bounds (now injectable limits) and three finalizer refusals. - CLAUDE.md and AGENTS.md name 9a.1 as the candidate build and finalizer; the ci.yml pack step comment mirrors release.yml; the register's post-closure CI record now matches the runs, and the 9a.1 checkpoint records round 1. Gates: lint:tools, typecheck:tools and every pnpm run ci step outside the three control families (whose captured sources are unchanged since a72c7ecd's full run), with the real producer in smoke:cli-package; a mutation run of nineteen mutants over the corrections kills all of them. Refs: ADR-0111 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 4 +- AGENTS.md | 2 +- CLAUDE.md | 2 +- docs/reference/cli/packaging.md | 10 +- .../phase-2.5.5-hardening-and-remediation.md | 42 +++++--- tools/cli-package/README.md | 6 +- tools/cli-package/candidate.mjs | 38 +++++-- tools/cli-package/candidate.test.mjs | 96 ++++++++++++++++- tools/cli-package/check.mjs | 13 +-- tools/cli-package/finalize.mjs | 19 +++- tools/cli-package/finalize.test.mjs | 50 +++++++++ tools/cli-package/process.mjs | 40 ++++++- tools/cli-package/produce.mjs | 39 +++++-- tools/cli-package/produce.test.mjs | 102 ++++++++++++++++++ 14 files changed, 404 insertions(+), 59 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ef328ad..6acae0e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -635,7 +635,9 @@ jobs: node-version-file: .nvmrc - name: Install (frozen lockfile) run: pnpm install --frozen-lockfile - # The runner builds inlined dependencies fresh, then prepack performs the sole final CLI build. + # The runner builds inlined dependencies fresh in an owned candidate tree, then prepack performs the sole + # final CLI build, where original-TS maps are built, validated, staged privately and discarded (Wave 3 + # Step 9a.1); only the tarball and its receipt leave this job. - name: Pack once and independently inspect the public package env: RELAVIUM_PACKAGE_SOURCE_SHA: ${{ github.sha }} diff --git a/AGENTS.md b/AGENTS.md index ba694a3a..b2f30f9b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -129,7 +129,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (release map/custody substrate, #46/ADR-0111) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-implementation-checkpoint--2026-10-22)). `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-implementation-checkpoint--2026-10-22)). `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. See [docs/roadmap/current.md](docs/roadmap/current.md) for live status. diff --git a/CLAUDE.md b/CLAUDE.md index a0f0c9ff..8817e9e3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -218,7 +218,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (release map/custody substrate, #46/ADR-0111) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-implementation-checkpoint--2026-10-22)). `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-implementation-checkpoint--2026-10-22)). `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. For live status, per-PR history, milestone dates, and open obligations, see the canonical diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 6cc9618a..fe4b571b 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -255,7 +255,8 @@ describes the implemented mechanism; independent acceptance is recorded in the directory it then runs the ordinary closure guard, as before. The CLI configuration emits external maps with `sourcesContent` disabled whenever `RELAVIUM_CLI_CANDIDATE_STAGING` is defined; prepack first refuses unless it names an - absolute, empty, owner-only directory outside the candidate, and then runs the finalizer + absolute, canonical, empty, owner-only directory outside the candidate (both compared by + their real paths), and then runs the finalizer in place of the ordinary guard. That guard refuses any map a build outside prepack leaves in `dist`. - **Finalizer.** Before any write it accounts for every emitted file (each runtime JS @@ -281,8 +282,11 @@ describes the implemented mechanism; independent acceptance is recorded in the unreviewed file in `dist`, and the five build configurations and the CLI configuration are pinned map-free outside a named candidate. -Private maps, the metafile and the hash record are discarded on every path after the -seal; no secure erasure is claimed. The private map manifest, actual packed CLI and engine +Private maps, the metafile, the hash record and the candidate tree are removed when the +producer succeeds, fails or is interrupted (SIGINT, SIGTERM, SIGHUP); an interrupt also stops +the build's child processes and empties the artifact directory, which was empty at the start. +A producer killed outright, a crash or a lost host can leave the owner-only root in the +temporary directory, and no secure erasure is claimed. The private map manifest, actual packed CLI and engine symbolication, `check:map-controls` and the `package_test` receipt flip are Step 9a.2; recipient custody, the release state controller and the workflow graph are Steps 9b–9d. Until those and Wave 4b's production recipient exist, the release workflow's publish job diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 18fcc54d..9296573c 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -9511,20 +9511,24 @@ was verified by the author alone, as the round cap provides; every finding it di [deferred-tasks.md](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18) are closed. Step 9a.1 follows. -**Post-closure CI correction, 2026-10-22.** `development` was pushed at `9eb38156`, and every CI job -passed except the separate engine coverage floor job, which `pnpm run ci` does not include: under -instrumented coverage, round 9's admitted-size linear-views case ran ten 8-MiB projections in one case -and passed its 15 s case deadline on the runner (3.7 s plain and 7.2 s under local coverage). `2a1f7359` -makes it one case per shape with the same inputs, assertions and deadline (1.1–2.8 s each under local -coverage; 221 core projection cases), and the local enforced coverage gate passes at llm 97.79 % lines -and 95.69 % branches, mcp 96.56 % and 91.29 %, core (measured) 96.71 % and 91.06 %. The author now also -runs the coverage gate before a push. CI at `2a1f7359` then passed every job but the filesystem causal -controls, whose export refused: each phase retains a manifest of the whole source graph (203,421 bytes -once PR95 grew it), and the 330 manifests of one export passed the unchanged 64-MiB bound by 20,066 -bytes. `4c3cafe9` runs that family as two exporting shards, as PR95's round 7 did for the secrets family -([filesystem controls README](../../../tools/fs-policy-controls/README.md)), and CI at `4c3cafe9` passed -every job, both shards of each control family, the coverage floor and the four installed-CLI consumers -included. +**Post-closure CI corrections, 2026-10-22.** `development` was pushed at `9eb38156` (run 39693684654). +Its separate engine coverage floor job, which `pnpm run ci` does not include, failed: under instrumented +coverage, round 9's admitted-size linear-views case ran ten 8-MiB projections in one case and passed its +15 s case deadline on the runner (3.7 s plain and 7.2 s under local coverage). The next push cancelled +that run before the filesystem controls and the second secrets shard finished, and its aggregating +`lint · typecheck · test` job failed at the step that requires the filesystem controls. `2a1f7359` (run +39698522813) makes the case one case per shape with the same inputs, assertions and deadline (1.1–2.8 s +each under local coverage; 221 core projection cases); its coverage job passed, and the local enforced +coverage gate passes at llm 97.79 % lines and 95.69 % branches, mcp 96.56 % and 91.29 %, core (measured) +96.71 % and 91.06 %. In that run the filesystem causal controls failed at export, and the aggregating +job failed with them: each phase retains a manifest of the whole source graph (203,421 bytes once PR95 +grew it), and the 330 manifests of one export passed the unchanged 64-MiB bound by 20,066 bytes, an +overflow already present at `9eb38156` because `2a1f7359` added no source file. `4c3cafe9` (run +39706480450) runs that family as two exporting shards, as PR95's round 7 did for the secrets family +([filesystem controls README](../../../tools/fs-policy-controls/README.md)), and that run passed all 17 +jobs, both shards of each control family, the coverage floor and the four installed-CLI consumers +included. The author now also runs the coverage gate before a push and checks each control family's +export arithmetic. ### Wave 3 Step 9a.1 implementation checkpoint — 2026-10-22 @@ -9566,6 +9570,16 @@ unchanged by this step, installs through the corepack pnpm shim in an empty home registry, and corepack hung for 45 s on this host while the same tree had passed it in the full run an hour earlier. +**Review rounds.** Each round is a NEW independent complete cumulative review of `4c3cafe9..HEAD`, +every reviewer in its own worktree detached at the reviewed head: plan and contract conformance, a +separate security sitting, producer and finalizer correctness (running the real producer), tests and +causal adequacy, integration and documentation, and a completeness critic. Every finding is +adversarially verified; the detailed reports stay local and Git-ignored. + +| Round | Reviewed head | Verdict | Verified findings | Main corrections | +| --- | --- | --- | --- | --- | +| 1 | `a72c7ecd` | NONCLEAN (plan lens clean) | 9 (2 medium, 6 low, 1 nit) | an interrupted producer (SIGINT, SIGTERM, SIGHUP) left its private maps and candidate tree in the temporary directory while its detached build ran on, so the producer now stops the child trees `runCommand` tracks and removes the private root, the build home and the artifact directory's contents before exiting with the signal's code (a producer killed outright is a named limit); private staging must be canonical and outside the candidate by real path; the global-excludes wiring, the producer environment, the placement check's artifact half, the candidate's bounds and three finalizer refusals gained killing tests; the status lines, the CI pack-step comment and the post-closure CI record were corrected | + **Remaining gates.** The increment is not independently accepted. Two NEW consecutive clean complete cumulative rounds after the last material fix and a separate security sitting remain (the Acceptance row of the 2026-10-19 rulings). Then `development` is pushed and its CI observed, and the PR #95 diff --git a/tools/cli-package/README.md b/tools/cli-package/README.md index abb5a760..9614bc04 100644 --- a/tools/cli-package/README.md +++ b/tools/cli-package/README.md @@ -37,8 +37,10 @@ controls exclude links, traversal, duplicates, extra files, maps and private bui inventories. This deliberately supports the pinned packer's dialect, not general tar. The resulting public `package-test-receipt.json` records `compiled_package_test`. -The candidate's maps are built, validated and then discarded with the rest of the -private staging; the receipt does **not** certify symbolication, a map manifest, +The candidate's maps are built, validated and then removed with the rest of the +private staging, also when the producer is interrupted (SIGINT, SIGTERM, SIGHUP), which +stops the build first; a producer killed outright can leave its owner-only root in the +temporary directory. The receipt does **not** certify symbolication, a map manifest, encrypted custody or a production release candidate (Steps 9a.2–9d). No private map, metafile or per-source inventory enters the public artifact. The release workflow's `release-guard.mjs` refuses every artifact until verified release custody exists: it diff --git a/tools/cli-package/candidate.mjs b/tools/cli-package/candidate.mjs index 864b57cf..d3a02769 100644 --- a/tools/cli-package/candidate.mjs +++ b/tools/cli-package/candidate.mjs @@ -104,6 +104,22 @@ export async function globalExcludesFile({ repository, environment, run = runCom return path !== undefined && existsSync(path) && statSync(path).isFile() ? path : undefined; } +/** + * The environment of a producer that a caller has isolated: `base` plus the dependency store and, when + * the caller found one, the developer's global Git excludes file, both found in the caller's own + * environment (`dependencyStore`, `globalExcludesFile`). + * @param {Record} base + * @param {{store: string, excludes?: string}} found + * @returns {Record} + */ +export function producerEnvironment(base, { store, excludes }) { + return { + ...base, + [PACKAGE_STORE_ENV]: store, + ...(excludes === undefined ? {} : { [GIT_EXCLUDES_ENV]: excludes }), + }; +} + /** @param {string | undefined} excludes @returns {string[]} */ function excludesArguments(excludes) { return excludes === undefined ? [] : ['-c', `core.excludesFile=${excludes}`]; @@ -137,8 +153,14 @@ export async function sourceIdentity(repository, run = runCommand, excludes = un * (`globalExcludesFile`). Ignored output, caches, dependencies and private notes never enter a * candidate. * @param {string} repository @param {typeof runCommand} [run] @param {string} [excludes] + * @param {{files: number}} [limits] the bounds (`CANDIDATE_LIMITS`; smaller ones in tests) */ -export async function candidateFiles(repository, run = runCommand, excludes = undefined) { +export async function candidateFiles( + repository, + run = runCommand, + excludes = undefined, + limits = CANDIDATE_LIMITS, +) { const options = { cwd: repository, env: isolatedEnvironment(repository) }; const listed = gitPaths( ( @@ -160,10 +182,7 @@ export async function candidateFiles(repository, run = runCommand, excludes = un gitPaths((await run('git', ['ls-files', '-z', '--deleted'], options)).stdout), ); const files = [...new Set(listed)].filter((path) => !deleted.has(path)).sort(); - assert.ok( - files.length > 0 && files.length <= CANDIDATE_LIMITS.files, - 'PACKED_CLI_CANDIDATE_FILES', - ); + assert.ok(files.length > 0 && files.length <= limits.files, 'PACKED_CLI_CANDIDATE_FILES'); return files; } @@ -171,9 +190,10 @@ export async function candidateFiles(repository, run = runCommand, excludes = un * Copy each listed ordinary file byte-for-byte into the empty `destination`. Links, special files, * escaping or noncanonical paths and oversized trees refuse; only the executable bit is kept, as an * exact 0755 or 0644 mode whatever the process umask. - * @param {{repository: string, destination: string, files: readonly string[]}} options + * @param {{repository: string, destination: string, files: readonly string[], + * limits?: {fileBytes: number, totalBytes: number}}} options `limits` defaults to `CANDIDATE_LIMITS` */ -export function copyCandidateTree({ repository, destination, files }) { +export function copyCandidateTree({ repository, destination, files, limits = CANDIDATE_LIMITS }) { assert.ok( isAbsolute(destination) && readdirSync(destination).length === 0, 'PACKED_CLI_CANDIDATE_ROOT', @@ -189,9 +209,9 @@ export function copyCandidateTree({ repository, destination, files }) { ); const source = join(repository, ...path.split('/')); const executable = (lstatSync(source).mode & 0o111) !== 0; - const bytes = readOrdinaryFile(source, CANDIDATE_LIMITS.fileBytes); + const bytes = readOrdinaryFile(source, limits.fileBytes); total += bytes.length; - assert.ok(total <= CANDIDATE_LIMITS.totalBytes, 'PACKED_CLI_CANDIDATE_BYTES'); + assert.ok(total <= limits.totalBytes, 'PACKED_CLI_CANDIDATE_BYTES'); const target = join(destination, ...path.split('/')); mkdirSync(dirname(target), { recursive: true, mode: 0o700 }); const mode = executable ? 0o755 : 0o644; diff --git a/tools/cli-package/candidate.test.mjs b/tools/cli-package/candidate.test.mjs index 3dd44786..30d8df5c 100644 --- a/tools/cli-package/candidate.test.mjs +++ b/tools/cli-package/candidate.test.mjs @@ -22,6 +22,7 @@ import { globalExcludesFile, installCandidateDependencies, prepareCandidate, + producerEnvironment, sourceIdentity, verifyCandidateLinks, } from './candidate.mjs'; @@ -475,7 +476,7 @@ test('a candidate build leaves no build cache behind', () => { }); /** A source repository plus fake git and pnpm ports; the fake install creates the workspace links. */ -function preparation({ statusAfterCopy = '', headAfterCopy = SHA } = {}) { +function preparation({ statusAfterCopy = '', headAfterCopy = SHA, excludes } = {}) { const repository = fresh('source'); write(join(repository, 'package.json'), '{}\n'); for (const name of ['shared', 'llm', 'core', 'db', 'mcp']) { @@ -491,6 +492,8 @@ function preparation({ statusAfterCopy = '', headAfterCopy = SHA } = {}) { const events = []; let statuses = 0; let heads = 0; + // With an excludes file, the listing and both status calls answer only when they carry it. + const prefix = excludes === undefined ? '' : `-c core.excludesFile=${excludes} `; const run = async (command, args) => { const key = args.join(' '); events.push(`git ${key}`); @@ -498,11 +501,11 @@ function preparation({ statusAfterCopy = '', headAfterCopy = SHA } = {}) { heads += 1; return { stdout: `${heads === 1 ? SHA : headAfterCopy}\n` }; } - if (key === 'status --porcelain=v1 -z --untracked-files=all') { + if (key === `${prefix}status --porcelain=v1 -z --untracked-files=all`) { statuses += 1; return { stdout: statuses === 1 ? '' : statusAfterCopy }; } - if (key === 'ls-files -z --cached --others --exclude-standard') + if (key === `${prefix}ls-files -z --cached --others --exclude-standard`) return { stdout: files.join('\0') }; if (key === 'ls-files -z --deleted') return { stdout: '' }; throw new Error(`unexpected git ${key}`); @@ -576,3 +579,90 @@ test('a source change during the copy refuses before anything is installed', asy ); } }); + +test("candidate preparation applies the developer's global excludes to every listing and status", async () => { + const ports = preparation({ excludes: '/developer/ignore' }); + const identity = await prepareCandidate({ + repository: ports.repository, + candidate: ports.candidate, + environment: {}, + home: ports.home, + env: {}, + excludes: '/developer/ignore', + run: ports.run, + pnpm: ports.pnpm, + }); + assert.deepEqual(identity, { source: SHA, dirty: false, files: 6 }); + assert.deepEqual( + ports.events.filter((event) => event.includes('status') || event.includes('--cached')), + [ + 'git -c core.excludesFile=/developer/ignore status --porcelain=v1 -z --untracked-files=all', + 'git -c core.excludesFile=/developer/ignore ls-files -z --cached --others --exclude-standard', + 'git -c core.excludesFile=/developer/ignore status --porcelain=v1 -z --untracked-files=all', + ], + ); +}); + +test('an isolated producer receives the store and, when found, the excludes file', () => { + assert.deepEqual(producerEnvironment({ HOME: '/isolated' }, { store: '/store' }), { + HOME: '/isolated', + RELAVIUM_PACKAGE_STORE_DIR: '/store', + }); + assert.deepEqual( + producerEnvironment({ HOME: '/isolated' }, { store: '/store', excludes: '/developer/ignore' }), + { + HOME: '/isolated', + RELAVIUM_PACKAGE_STORE_DIR: '/store', + RELAVIUM_GIT_EXCLUDES_FILE: '/developer/ignore', + }, + ); +}); + +test('the candidate refuses a file list or a tree over its bounds', async () => { + const listing = git({ + 'ls-files -z --cached --others --exclude-standard': 'a.ts\0b.ts\0c.ts\0', + 'ls-files -z --deleted': '', + }); + assert.deepEqual(await candidateFiles('/repository', listing.run, undefined, { files: 3 }), [ + 'a.ts', + 'b.ts', + 'c.ts', + ]); + await assert.rejects( + candidateFiles('/repository', listing.run, undefined, { files: 2 }), + /CANDIDATE_FILES/u, + ); + const repository = fresh('bounded'); + write(join(repository, 'a.ts'), 'abcdef'); + write(join(repository, 'b.ts'), 'ghijkl'); + const files = ['a.ts', 'b.ts']; + assert.deepEqual( + copyCandidateTree({ + repository, + destination: fresh('copy'), + files, + limits: { fileBytes: 6, totalBytes: 12 }, + }), + { files: 2, bytes: 12 }, + ); + assert.throws( + () => + copyCandidateTree({ + repository, + destination: fresh('copy'), + files, + limits: { fileBytes: 6, totalBytes: 11 }, + }), + /CANDIDATE_BYTES/u, + ); + assert.throws( + () => + copyCandidateTree({ + repository, + destination: fresh('copy'), + files, + limits: { fileBytes: 5, totalBytes: 12 }, + }), + /FILE_KIND_OR_BOUND/u, + ); +}); diff --git a/tools/cli-package/check.mjs b/tools/cli-package/check.mjs index 1db2bfd8..b5ec789c 100644 --- a/tools/cli-package/check.mjs +++ b/tools/cli-package/check.mjs @@ -4,12 +4,7 @@ import { mkdtempSync, readFileSync, realpathSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { - GIT_EXCLUDES_ENV, - PACKAGE_STORE_ENV, - dependencyStore, - globalExcludesFile, -} from './candidate.mjs'; +import { dependencyStore, globalExcludesFile, producerEnvironment } from './candidate.mjs'; import { isolatedEnvironment, runCommand } from './process.mjs'; assert.equal(process.argv.length, 2, 'no arguments required'); @@ -26,11 +21,7 @@ try { const options = { env: isolatedEnvironment(root), timeoutMs: 600_000 }; await runCommand(process.execPath, [join(directory, 'pack.mjs'), artifact], { ...options, - env: { - ...options.env, - [PACKAGE_STORE_ENV]: store, - ...(excludes === undefined ? {} : { [GIT_EXCLUDES_ENV]: excludes }), - }, + env: producerEnvironment(options.env, { store, excludes }), }); await runCommand(process.execPath, [join(directory, 'consume.mjs'), artifact, reports], options); const record = JSON.parse(readFileSync(join(reports, 'installed-consumer.json'), 'utf8')); diff --git a/tools/cli-package/finalize.mjs b/tools/cli-package/finalize.mjs index 9eab4723..68cc60ee 100644 --- a/tools/cli-package/finalize.mjs +++ b/tools/cli-package/finalize.mjs @@ -19,6 +19,7 @@ import { mkdirSync, openSync, readdirSync, + realpathSync, renameSync, unlinkSync, writeSync, @@ -291,19 +292,31 @@ function finalMap(repository, jsName, map, expected) { } /** - * A private staging directory must be absolute, empty, an ordinary directory outside the candidate - * repository and, where POSIX modes are observable, owner-only and owned by this process. + * A private staging directory must be absolute and canonical (no link anywhere in its path), empty, an + * ordinary directory outside the candidate repository and, where POSIX modes are observable, owner-only + * and owned by this process. The repository is compared by its real path too, so no alias of either + * side lets staging sit inside the candidate. * @param {string} staging @param {string} repository */ export function assertPrivateStaging(staging, repository) { if (typeof staging !== 'string' || !isAbsolute(staging)) refuse('STAGING_PATH'); let info; + let real; + let root; try { info = lstatSync(staging); + real = realpathSync(staging); + root = realpathSync(repository); } catch { refuse('STAGING_PATH'); } - if (!info.isDirectory() || within(repository, staging) || within(staging, repository)) { + if ( + !info.isDirectory() || + real !== staging || + within(root, real) || + within(real, root) || + within(repository, staging) + ) { refuse('STAGING_PATH'); } if (readdirSync(staging).length !== 0) refuse('STAGING_NOT_EMPTY'); diff --git a/tools/cli-package/finalize.test.mjs b/tools/cli-package/finalize.test.mjs index df9e96d2..f9a2c2b3 100644 --- a/tools/cli-package/finalize.test.mjs +++ b/tools/cli-package/finalize.test.mjs @@ -287,6 +287,16 @@ for (const [label, change, code] of [ }); } +test('refuses an upstream map source that exists but is not a file', async () => { + const candidate = await fixture(); + mkdirSync(join(candidate.core, 'src', 'dir.ts')); + editJson( + join(candidate.core, 'dist', 'index.js.map'), + (map) => (map.sources = ['../src/dir.ts']), + ); + refusesBeforeWriting(candidate, 'MAP_SOURCE_MISSING'); +}); + test('refuses an unparsable upstream map and an inexact upstream map link', async () => { const unparsable = await fixture(); writeFileSync(join(unparsable.core, 'dist', 'index.js.map'), '{'); @@ -375,6 +385,14 @@ test('accounts for every emitted file and every consumed input', async () => { }; }); refusesBeforeWriting(foreign, 'UNSUPPORTED_INPUT'); + // Only the five inlined engine packages and the CLI's own TypeScript are supported inputs. + for (const input of ['../../packages/ui/dist/x.js', 'src/types.d.ts']) { + const unsupported = await fixture(); + editJson(join(unsupported.dist, 'metafile-esm.json'), (metafile) => { + metafile.inputs[input] = { bytes: 1, imports: [], format: 'esm' }; + }); + refusesBeforeWriting(unsupported, 'UNSUPPORTED_INPUT'); + } if (posixModes) { const linked = await fixture(); symlinkSync(join(linked.dist, 'index.js'), join(linked.dist, 'alias.js')); @@ -427,6 +445,38 @@ test('requires an empty private staging directory outside the candidate', async () => finalizeCandidate({ repository: candidate.root, staging: link }), (error) => error instanceof CandidateFinalizationError && error.code === 'STAGING_PATH', ); + // A real owner-only directory inside the candidate, named through a linked parent outside it. + const inside = join(candidate.cli, 'private'); + mkdirSync(inside, { mode: 0o700 }); + chmodSync(inside, 0o700); + const alias = join(scratch, `alias-parent-${count}`); + symlinkSync(candidate.cli, alias); + assert.throws( + () => finalizeCandidate({ repository: candidate.root, staging: join(alias, 'private') }), + (error) => error instanceof CandidateFinalizationError && error.code === 'STAGING_PATH', + ); + // The same real directory refuses when the repository itself is named through a link. + const aliasRoot = join(scratch, `alias-root-${count}`); + symlinkSync(candidate.root, aliasRoot); + assert.throws( + () => finalizeCandidate({ repository: aliasRoot, staging: inside }), + (error) => error instanceof CandidateFinalizationError && error.code === 'STAGING_PATH', + ); + rmSync(inside, { recursive: true }); + // Staging must be named canonically even outside the candidate. + const outside = join(scratch, `outside-staging-${count}`); + mkdirSync(outside, { mode: 0o700 }); + chmodSync(outside, 0o700); + const outsideAlias = join(scratch, `outside-alias-${count}`); + symlinkSync(scratch, outsideAlias); + assert.throws( + () => + finalizeCandidate({ + repository: candidate.root, + staging: join(outsideAlias, `outside-staging-${count}`), + }), + (error) => error instanceof CandidateFinalizationError && error.code === 'STAGING_PATH', + ); } }); diff --git a/tools/cli-package/process.mjs b/tools/cli-package/process.mjs index 89127b3e..8cee4d14 100644 --- a/tools/cli-package/process.mjs +++ b/tools/cli-package/process.mjs @@ -1,6 +1,6 @@ /** Native tooling boundary. Never starts a shell or inherits credentials/config/preloads. */ import assert from 'node:assert/strict'; -import { spawn } from 'node:child_process'; +import { spawn, spawnSync } from 'node:child_process'; import { Buffer } from 'node:buffer'; import { existsSync, lstatSync, readFileSync, realpathSync, statSync } from 'node:fs'; import { delimiter, extname, isAbsolute, join } from 'node:path'; @@ -58,6 +58,41 @@ export function resolvePnpm(path = process.env.PATH ?? '', platform = process.pl throw new Error('PACKED_CLI_PINNED_PNPM_NOT_FOUND'); } +/** Every child tree a `runCommand` call still owns, so that an interrupted tool can stop them first. */ +const liveChildren = new Set(); + +/** + * Synchronously kill every child tree that `runCommand` still owns: a POSIX process group, or a + * Windows tree through taskkill. For a signal handler that must stop detached builds before it exits. + */ +export function stopLiveCommands() { + for (const child of liveChildren) { + if (child.grouped) { + try { + process.kill(-child.pid, 'SIGKILL'); + } catch { + // The group already exited. + } + continue; + } + const systemRoot = process.env.SystemRoot; + if (systemRoot && isAbsolute(systemRoot)) { + spawnSync( + join(systemRoot, 'System32', 'taskkill.exe'), + ['/PID', String(child.pid), '/T', '/F'], + { shell: false, stdio: 'ignore' }, + ); + } else { + try { + process.kill(child.pid, 'SIGKILL'); + } catch { + // The child already exited. + } + } + } + liveChildren.clear(); +} + export function runCommand(command, args, options = {}) { const { timeoutMs = 300_000, maxBytes = 8 * 1024 * 1024, ...spawnOptions } = options; assert.ok(Number.isSafeInteger(timeoutMs) && timeoutMs > 0); @@ -70,6 +105,8 @@ export function runCommand(command, args, options = {}) { shell: false, stdio: ['ignore', 'pipe', 'pipe'], }); + const live = { pid: child.pid, grouped }; + if (child.pid !== undefined) liveChildren.add(live); let bytes = 0, fault, closed = false, @@ -118,6 +155,7 @@ export function runCommand(command, args, options = {}) { }); child.once('close', async (code, signal) => { closed = true; + liveChildren.delete(live); clearTimeout(timer); if (killer && killer.exitCode === null && killer.signalCode === null) await new Promise((done) => killer.once('close', done)); diff --git a/tools/cli-package/produce.mjs b/tools/cli-package/produce.mjs index e53c05c4..46f7ea35 100644 --- a/tools/cli-package/produce.mjs +++ b/tools/cli-package/produce.mjs @@ -2,19 +2,24 @@ * The package producer (ADR-0111, Wave 3 Step 9a.1): owned candidate copy, fresh mapped dependency * emit, sole CLI prepack with candidate finalization, independent seal. It never builds in the * developer's workspace and never calls a cached build. The private maps, metafile and hash record - * are staged owner-only outside the artifact directory and discarded on every path; only the tarball - * and its public receipt are published. `pack.mjs` is its command-line entry point; the ports are - * tooling seams for tests, never shipped product hooks. + * are staged owner-only outside the artifact directory and removed when production succeeds, fails or + * is interrupted (SIGINT, SIGTERM, SIGHUP); a producer killed outright, a crash or a lost host can + * leave its owner-only root in the temporary directory, and no secure erasure is claimed. Only the + * tarball and its public receipt are published. `pack.mjs` is its command-line entry point; the ports + * are tooling seams for tests, never shipped product hooks. */ import assert from 'node:assert/strict'; import { chmodSync, mkdirSync, mkdtempSync, readdirSync, realpathSync, rmSync } from 'node:fs'; -import { tmpdir } from 'node:os'; +import { constants, tmpdir } from 'node:os'; import { isAbsolute, join, relative } from 'node:path'; import { assertNoBuildCache, globalExcludesFile, prepareCandidate } from './candidate.mjs'; -import { isolatedEnvironment, runPnpm } from './process.mjs'; +import { isolatedEnvironment, runPnpm, stopLiveCommands } from './process.mjs'; import { sealPackage } from './seal.mjs'; import { emitFreshPackage } from './pipeline.mjs'; +/** The interrupts after which the producer still removes everything private before it exits. */ +const INTERRUPTS = Object.freeze(['SIGINT', 'SIGTERM', 'SIGHUP']); + /** @param {string} parent @param {string} child */ function within(parent, child) { const path = relative(parent, child); @@ -24,8 +29,10 @@ function within(parent, child) { /** * Produce one package-test artifact from the original checkout `repository` into the empty * `destination`. The candidate tree and its private staging are siblings in one owner-only temporary - * root that shares no path with the artifact directory or the checkout, and that root is removed - * whether production succeeds or fails; no secure erasure is claimed. + * root that shares no path with the artifact directory or the checkout. That root and the build home + * are removed whether production succeeds or fails; an interrupt also stops the build's child trees + * and empties the artifact directory, which was empty at the start, before the process exits with + * the signal's code. * @param {{repository: string, destination: string, environment: Record, * temporary?: string, ports?: {prepare?: typeof prepareCandidate, emit?: typeof emitFreshPackage, * seal?: typeof sealPackage, noBuildCache?: typeof assertNoBuildCache, @@ -50,6 +57,19 @@ export async function producePackage({ assert.deepEqual(readdirSync(destination), [], 'PACKED_CLI_FRESH_DESTINATION_REQUIRED'); const home = realpathSync(mkdtempSync(join(temporary, 'relavium-package-build-'))); let root; + const discard = () => { + if (root !== undefined) rmSync(root, { recursive: true, force: true }); + rmSync(home, { recursive: true, force: true }); + }; + const interrupt = (signal) => { + stopLiveCommands(); + discard(); + for (const name of readdirSync(destination)) { + rmSync(join(destination, name), { recursive: true, force: true }); + } + process.exit(128 + (constants.signals[signal] ?? 0)); + }; + for (const signal of INTERRUPTS) process.on(signal, interrupt); try { root = realpathSync(mkdtempSync(join(temporary, 'relavium-package-candidate-'))); chmodSync(root, 0o700); @@ -91,8 +111,7 @@ export async function producePackage({ }); return { receipt, identity }; } finally { - // Private maps never outlive the producer. - if (root !== undefined) rmSync(root, { recursive: true, force: true }); - rmSync(home, { recursive: true, force: true }); + for (const signal of INTERRUPTS) process.off(signal, interrupt); + discard(); } } diff --git a/tools/cli-package/produce.test.mjs b/tools/cli-package/produce.test.mjs index 60545832..274eb52d 100644 --- a/tools/cli-package/produce.test.mjs +++ b/tools/cli-package/produce.test.mjs @@ -1,8 +1,10 @@ import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; import { existsSync, mkdirSync, mkdtempSync, + readFileSync, readdirSync, realpathSync, rmSync, @@ -12,6 +14,7 @@ import { import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { after, test } from 'node:test'; +import { setTimeout as delay } from 'node:timers/promises'; import { assertNoBuildCache } from './candidate.mjs'; import { producePackage } from './produce.mjs'; @@ -167,6 +170,22 @@ test('private staging never shares a path with the checkout or the artifact', as ); assert.equal(seen.prepare, undefined); assert.deepEqual(leftovers(inside), []); + // So would a temporary directory inside the artifact directory. + const artifact = join(fresh('artifact-parent'), 'artifact'); + mkdirSync(artifact); + const { seen: inArtifact, ports: artifactFakes } = ports(); + await assert.rejects( + producePackage({ + repository, + destination: artifact, + environment: {}, + temporary: artifact, + ports: artifactFakes, + }), + /PRIVATE_STAGING_PLACEMENT/u, + ); + assert.equal(inArtifact.prepare, undefined); + assert.deepEqual(readdirSync(artifact), []); // A non-empty artifact directory refuses before anything private exists. const temporary = fresh('tmp'); const destination = fresh('artifact'); @@ -177,3 +196,86 @@ test('private staging never shares a path with the checkout or the artifact', as ); assert.deepEqual(leftovers(temporary), []); }); + +/** A producer run whose build is a long-lived child, as `pnpm pack` is, behind fake ports. */ +function interruptedProducer(temporary, destination, pidFile) { + const url = (name) => JSON.stringify(new URL(name, import.meta.url).href); + const script = ` +import { writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { producePackage } from ${url('./produce.mjs')}; +import { isolatedEnvironment, runCommand } from ${url('./process.mjs')}; +await producePackage({ + repository: ${JSON.stringify(join(temporary, 'checkout'))}, + destination: ${JSON.stringify(destination)}, + environment: {}, + temporary: ${JSON.stringify(temporary)}, + ports: { + pnpm: async () => ({ stdout: '9.12.3\\n', stderr: '' }), + excludesFile: async () => undefined, + prepare: async () => ({ source: '${SHA}', dirty: false, files: 1 }), + emit: async ({ staging, destination }) => { + writeFileSync(join(staging, 'index.js.map'), 'private map'); + writeFileSync(join(destination, 'relavium-0.1.1.tgz'), 'unsealed'); + await runCommand(process.execPath, ['-e', ${JSON.stringify( + `require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, String(process.pid)); setInterval(() => {}, 1000);`, + )}], { env: isolatedEnvironment(${JSON.stringify(temporary)}), timeoutMs: 60_000 }); + }, + }, +}); +`; + return spawn(process.execPath, ['--input-type=module', '-e', script], { + stdio: ['ignore', 'pipe', 'pipe'], + }); +} + +function alive(pid) { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +for (const [signal, code] of [ + ['SIGINT', 130], + ['SIGTERM', 143], +]) { + test( + `an interrupted producer (${signal}) stops its build and removes everything private`, + { skip: !posixModes }, + async () => { + const temporary = fresh('tmp'); + mkdirSync(join(temporary, 'checkout')); + const destination = join(fresh('artifact-parent'), 'artifact'); + const pidFile = join(fresh('pid'), 'sleeper.pid'); + const child = interruptedProducer(temporary, destination, pidFile); + let sleeper; + try { + for ( + let waited = 0; + !existsSync(pidFile) || readFileSync(pidFile, 'utf8') === ''; + waited += 25 + ) { + assert.ok(waited < 20_000, 'the fake build never started'); + await delay(25); + } + sleeper = Number(readFileSync(pidFile, 'utf8')); + assert.ok(alive(sleeper)); + const exited = new Promise((resolve) => child.once('exit', resolve)); + child.kill(signal); + assert.equal(await exited, code); + for (let waited = 0; alive(sleeper); waited += 25) { + assert.ok(waited < 5_000, 'the build child outlived the interrupt'); + await delay(25); + } + assert.deepEqual(leftovers(temporary), []); + assert.deepEqual(readdirSync(destination), []); + } finally { + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); + if (sleeper !== undefined && alive(sleeper)) process.kill(sleeper, 'SIGKILL'); + } + }, + ); +} From 184562bddb3edbde92c9f8202c6fc47118f09fd8 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Thu, 22 Oct 2026 08:00:58 +0300 Subject: [PATCH 24/97] fix(repo): address the second Step 9a.1 review round Round 2 (at 8ddb7ec2) reported twelve verified findings, four medium; its plan and integration lenses were clean. - The producer's finally removed its interrupt handlers before the final removal, so an interrupt during it ended the process with the private root half removed. The handlers now stay until everything private is gone; the removal is a port, and a test interrupts it. - SIGHUP, the handlers' removal and a whole build tree (a grandchild in the build's process group) are tested. - Path containment is decided by segments in one shared helper (paths.mjs); '..private' is inside. - Private staging and the producer's placement compare the real paths and letter case the filesystem stores (realpathSync.native). - pnpm smoke:cli-package forwards an interrupt to the producer and the consumer it started (forwardInterrupts) instead of ending first. - The finalizer gives its maps directory an exact mode; staged and public modes are tested under umask 0277; further finalizer and staged-closure refusals gained killing tests. - The ADR-0051 note links ADR-0111, and the 9a.1 checkpoint records round 2. Gates: lint:tools, typecheck:tools and every pnpm run ci step outside the three control families (their captured sources are unchanged), with the real producer in smoke:cli-package; a mutation run of eleven mutants over the corrections kills all of them. Refs: ADR-0111, ADR-0051 Co-Authored-By: Claude Opus 5.5 --- ...distribution-thin-bundle-private-engine.md | 2 +- docs/reference/cli/packaging.md | 7 +- .../phase-2.5.5-hardening-and-remediation.md | 1 + tools/bundle-closure/closure.test.mjs | 5 + tools/cli-package/README.md | 5 +- tools/cli-package/candidate.mjs | 9 +- tools/cli-package/candidate.test.mjs | 26 ++-- tools/cli-package/check.mjs | 13 +- tools/cli-package/finalize.mjs | 25 ++-- tools/cli-package/finalize.test.mjs | 87 ++++++++++++ tools/cli-package/paths.mjs | 12 ++ tools/cli-package/prepack.test.mjs | 1 + tools/cli-package/process.mjs | 33 +++++ tools/cli-package/process.test.mjs | 83 +++++++++++- tools/cli-package/produce.mjs | 31 ++--- tools/cli-package/produce.test.mjs | 127 +++++++++++++++--- 16 files changed, 394 insertions(+), 73 deletions(-) create mode 100644 tools/cli-package/paths.mjs diff --git a/docs/decisions/0051-cli-distribution-thin-bundle-private-engine.md b/docs/decisions/0051-cli-distribution-thin-bundle-private-engine.md index 732ae511..ae0afef3 100644 --- a/docs/decisions/0051-cli-distribution-thin-bundle-private-engine.md +++ b/docs/decisions/0051-cli-distribution-thin-bundle-private-engine.md @@ -152,4 +152,4 @@ intended precedent for the desktop (`.dmg`) and VS Code (`.vsix`) surfaces. > Amended 2026-10-16: [ADR-0111](0111-cli-meta-operations-use-a-lazy-packaged-runtime.md) is accepted by the maintainer. Only the single-output constraint is partially replaced by finite engine-inlined runtime chunks with complete closure and packed-platform checks. Licensing rather than source secrecy, private engine packages and no public plaintext source maps remain accepted. Exact release-map encrypted custody is a distribution policy, not secrecy for this public source repository. The approved [execution plan](../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) owns implementation and review gates; this note does not claim implementation landing. -> Amended 2026-10-22: Wave 3 Step 9a.1 implements the candidate build. A candidate build — the CLI build with `RELAVIUM_CLI_CANDIDATE_STAGING` defined, which the prepack lifecycle runs only after it validates that directory as private staging — emits external maps without embedded sources, and its finalizer stages them outside the package before anything is hashed or packed, so the published artifact still carries no map or map link; the finalizer, the closure guard and the archive inspector each refuse one. "The publish build sets `sourcemap: false`" now describes ordinary builds. The [packaging contract](../reference/cli/packaging.md#candidate-build-and-finalizer-implementation-boundary--step-9a1) records the implemented boundary; this note claims no symbolication, custody or release. +> Amended 2026-10-22: Wave 3 Step 9a.1 implements the candidate build under [ADR-0111](0111-cli-meta-operations-use-a-lazy-packaged-runtime.md). A candidate build — the CLI build with `RELAVIUM_CLI_CANDIDATE_STAGING` defined, which the prepack lifecycle runs only after it validates that directory as private staging — emits external maps without embedded sources, and its finalizer stages them outside the package before anything is hashed or packed, so the published artifact still carries no map or map link; the finalizer, the closure guard and the archive inspector each refuse one. "The publish build sets `sourcemap: false`" now describes ordinary builds. The [packaging contract](../reference/cli/packaging.md#candidate-build-and-finalizer-implementation-boundary--step-9a1) records the implemented boundary; this note claims no symbolication, custody or release. diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index fe4b571b..e7f5e995 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -256,7 +256,7 @@ describes the implemented mechanism; independent acceptance is recorded in the emits external maps with `sourcesContent` disabled whenever `RELAVIUM_CLI_CANDIDATE_STAGING` is defined; prepack first refuses unless it names an absolute, canonical, empty, owner-only directory outside the candidate (both compared by - their real paths), and then runs the finalizer + the real paths and letter case the filesystem stores), and then runs the finalizer in place of the ordinary guard. That guard refuses any map a build outside prepack leaves in `dist`. - **Finalizer.** Before any write it accounts for every emitted file (each runtime JS @@ -284,8 +284,9 @@ describes the implemented mechanism; independent acceptance is recorded in the Private maps, the metafile, the hash record and the candidate tree are removed when the producer succeeds, fails or is interrupted (SIGINT, SIGTERM, SIGHUP); an interrupt also stops -the build's child processes and empties the artifact directory, which was empty at the start. -A producer killed outright, a crash or a lost host can leave the owner-only root in the +the build's child processes and empties the artifact directory, which was empty at the start, +and the local `pnpm smoke:cli-package` forwards an interrupt to the producer it started rather than +ending first. A producer killed outright, a crash or a lost host can leave the owner-only root in the temporary directory, and no secure erasure is claimed. The private map manifest, actual packed CLI and engine symbolication, `check:map-controls` and the `package_test` receipt flip are Step 9a.2; recipient custody, the release state controller and the workflow graph are Steps 9b–9d. diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 9296573c..6be8f0aa 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -9579,6 +9579,7 @@ adversarially verified; the detailed reports stay local and Git-ignored. | Round | Reviewed head | Verdict | Verified findings | Main corrections | | --- | --- | --- | --- | --- | | 1 | `a72c7ecd` | NONCLEAN (plan lens clean) | 9 (2 medium, 6 low, 1 nit) | an interrupted producer (SIGINT, SIGTERM, SIGHUP) left its private maps and candidate tree in the temporary directory while its detached build ran on, so the producer now stops the child trees `runCommand` tracks and removes the private root, the build home and the artifact directory's contents before exiting with the signal's code (a producer killed outright is a named limit); private staging must be canonical and outside the candidate by real path; the global-excludes wiring, the producer environment, the placement check's artifact half, the candidate's bounds and three finalizer refusals gained killing tests; the status lines, the CI pack-step comment and the post-closure CI record were corrected | +| 2 | `8ddb7ec2` | NONCLEAN (plan and integration lenses clean) | 12 (4 medium, 7 low, 1 nit; 9 distinct) | the producer removed its interrupt handlers before the final removal, so an interrupt during it ended the process with the private root half removed: the handlers now stay until everything private is gone, and a deterministic test interrupts that removal; SIGHUP, the handlers' removal and a whole build tree (a grandchild in the build's process group) are tested; path containment is decided by segments in one shared helper (`..private` is inside); private staging and the producer's placement compare the real paths and letter case the filesystem stores (`realpathSync.native`); the local `pnpm smoke:cli-package` forwards an interrupt to the producer and the consumer it started (`forwardInterrupts`); the finalizer creates its maps directory with an exact mode and its staged and public modes are tested under a umask that removes the owner's write bit; further finalizer and staged-closure refusals gained killing tests; the ADR-0051 note links ADR-0111 | **Remaining gates.** The increment is not independently accepted. Two NEW consecutive clean complete cumulative rounds after the last material fix and a separate security sitting remain (the Acceptance diff --git a/tools/bundle-closure/closure.test.mjs b/tools/bundle-closure/closure.test.mjs index 6af1b7b0..b195f85d 100644 --- a/tools/bundle-closure/closure.test.mjs +++ b/tools/bundle-closure/closure.test.mjs @@ -593,6 +593,11 @@ for (const [label, mutate, code] of [ (value) => (value.metafile.outputs['dist/index.js.map'].entryPoint = 'src/index.ts'), 'MAP_OUTPUT', ], + [ + 'map metadata with exports', + (value) => (value.metafile.outputs['dist/index.js.map'].exports = ['run']), + 'MAP_OUTPUT', + ], [ 'a nested map', (value) => diff --git a/tools/cli-package/README.md b/tools/cli-package/README.md index 9614bc04..bf2ebe49 100644 --- a/tools/cli-package/README.md +++ b/tools/cli-package/README.md @@ -39,8 +39,9 @@ inventories. This deliberately supports the pinned packer's dialect, not general The resulting public `package-test-receipt.json` records `compiled_package_test`. The candidate's maps are built, validated and then removed with the rest of the private staging, also when the producer is interrupted (SIGINT, SIGTERM, SIGHUP), which -stops the build first; a producer killed outright can leave its owner-only root in the -temporary directory. The receipt does **not** certify symbolication, a map manifest, +stops the build first, and `pnpm smoke:cli-package` forwards an interrupt to the producer +and the consumer it started instead of ending before them; a producer killed outright can +leave its owner-only root in the temporary directory. The receipt does **not** certify symbolication, a map manifest, encrypted custody or a production release candidate (Steps 9a.2–9d). No private map, metafile or per-source inventory enters the public artifact. The release workflow's `release-guard.mjs` refuses every artifact until verified release custody exists: it diff --git a/tools/cli-package/candidate.mjs b/tools/cli-package/candidate.mjs index d3a02769..c8da57a1 100644 --- a/tools/cli-package/candidate.mjs +++ b/tools/cli-package/candidate.mjs @@ -17,9 +17,10 @@ import { statSync, writeFileSync, } from 'node:fs'; -import { dirname, isAbsolute, join, relative, resolve } from 'node:path'; +import { dirname, isAbsolute, join, resolve } from 'node:path'; import { ENGINE_PACKAGES } from './finalize.mjs'; import { readOrdinaryFile } from './files.mjs'; +import { within } from './paths.mjs'; import { isolatedEnvironment, runCommand, runPnpm } from './process.mjs'; export const CANDIDATE_LIMITS = Object.freeze({ @@ -39,12 +40,6 @@ const STORE_DISCOVERY_VARIABLES = Object.freeze([ 'PNPM_HOME', ]); -/** @param {string} parent @param {string} child */ -function within(parent, child) { - const path = relative(parent, child); - return path === '' || (!path.startsWith('..') && !isAbsolute(path)); -} - /** @param {string} output NUL-separated Git path list */ function gitPaths(output) { return output.split('\0').filter((path) => path !== ''); diff --git a/tools/cli-package/candidate.test.mjs b/tools/cli-package/candidate.test.mjs index 30d8df5c..e8d6db20 100644 --- a/tools/cli-package/candidate.test.mjs +++ b/tools/cli-package/candidate.test.mjs @@ -374,17 +374,21 @@ test('the dependency store is discovered with store-location variables only', as test('store discovery refuses a private home inside the original checkout', async () => { const repository = fresh('checkout'); - await assert.rejects( - dependencyStore({ - repository, - environment: { HOME: '/home/someone' }, - home: join(repository, 'home'), - pnpm: async () => { - throw new Error('store discovery must not run'); - }, - }), - /PRIVATE_HOME/u, - ); + // A child whose name merely starts with two dots is inside the checkout too. + for (const home of [join(repository, 'home'), join(repository, '..home')]) { + await assert.rejects( + dependencyStore({ + repository, + environment: { HOME: '/home/someone' }, + home, + pnpm: async () => { + throw new Error('store discovery must not run'); + }, + }), + /PRIVATE_HOME/u, + home, + ); + } }); test('the candidate installs offline, frozen and without lifecycle scripts', async () => { diff --git a/tools/cli-package/check.mjs b/tools/cli-package/check.mjs index b5ec789c..c38d44a4 100644 --- a/tools/cli-package/check.mjs +++ b/tools/cli-package/check.mjs @@ -1,15 +1,19 @@ /** Local full-CI counterpart: one new candidate and one actual-host consumer, never a platform aggregate. */ import assert from 'node:assert/strict'; import { mkdtempSync, readFileSync, realpathSync, rmSync } from 'node:fs'; -import { tmpdir } from 'node:os'; +import { constants, tmpdir } from 'node:os'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { dependencyStore, globalExcludesFile, producerEnvironment } from './candidate.mjs'; -import { isolatedEnvironment, runCommand } from './process.mjs'; +import { forwardInterrupts, isolatedEnvironment, runCommand } from './process.mjs'; assert.equal(process.argv.length, 2, 'no arguments required'); const directory = fileURLToPath(new URL('.', import.meta.url)); const root = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-local-package-check-'))); +// The producer and the consumer run detached and clean up after themselves (the producer removes its +// private root on an interrupt), so an interrupt reaches them instead of ending this check first; the +// check root, which holds their temporary directories, goes once they have exited. +const interrupts = forwardInterrupts(); try { // The producer runs with an isolated HOME, so the store and the developer's global Git excludes // file are found here, from the caller's own location variables, and handed over explicitly. @@ -37,6 +41,11 @@ try { smokeCases: record.smoke.cases.length, }), ); +} catch (error) { + if (interrupts.received() === undefined) throw error; } finally { rmSync(root, { recursive: true, force: true }); + interrupts.dispose(); } +const signal = interrupts.received(); +if (signal !== undefined) process.exit(128 + (constants.signals[signal] ?? 0)); diff --git a/tools/cli-package/finalize.mjs b/tools/cli-package/finalize.mjs index 68cc60ee..42348268 100644 --- a/tools/cli-package/finalize.mjs +++ b/tools/cli-package/finalize.mjs @@ -12,6 +12,7 @@ */ import { Buffer } from 'node:buffer'; import { + chmodSync, closeSync, fchmodSync, fsyncSync, @@ -24,11 +25,12 @@ import { unlinkSync, writeSync, } from 'node:fs'; -import { basename, isAbsolute, join, posix, relative } from 'node:path'; +import { basename, isAbsolute, join, posix } from 'node:path'; import { TextDecoder, isDeepStrictEqual } from 'node:util'; import { checkBundleClosure } from '../bundle-closure/closure.mjs'; import { MODE_EVIDENCE, emittedInventory } from '../bundle-closure/inventory.mjs'; import { flatFiles, readOrdinaryFile, sha256 } from './files.mjs'; +import { within } from './paths.mjs'; /** * Names the private staging directory of a trusted candidate build. Only the package producer sets @@ -75,12 +77,6 @@ function isRecord(value) { /** The map-link trailer tsc and tsup write for an external map beside its JS file. */ export const mapTrailer = (/** @type {string} */ jsName) => `//# sourceMappingURL=${jsName}.map`; -/** @param {string} parent @param {string} child */ -function within(parent, child) { - const path = relative(parent, child); - return path === '' || (!path.startsWith('..') && !isAbsolute(path)); -} - /** @param {string} root @param {string} path repository-relative POSIX path */ function onDisk(root, path) { return join(root, ...path.split('/')); @@ -292,10 +288,11 @@ function finalMap(repository, jsName, map, expected) { } /** - * A private staging directory must be absolute and canonical (no link anywhere in its path), empty, an - * ordinary directory outside the candidate repository and, where POSIX modes are observable, owner-only - * and owned by this process. The repository is compared by its real path too, so no alias of either - * side lets staging sit inside the candidate. + * A private staging directory must be absolute and canonical (no link in its path and the letter case + * the filesystem stores, `realpathSync.native`), empty, an ordinary directory outside the candidate + * repository and, where POSIX modes are observable, owner-only and owned by this process. The + * repository is compared by its native real path too, so no alias of either side (a link or, on a + * case-insensitive filesystem, another letter case) lets staging sit inside the candidate. * @param {string} staging @param {string} repository */ export function assertPrivateStaging(staging, repository) { @@ -305,8 +302,8 @@ export function assertPrivateStaging(staging, repository) { let root; try { info = lstatSync(staging); - real = realpathSync(staging); - root = realpathSync(repository); + real = realpathSync.native(staging); + root = realpathSync.native(repository); } catch { refuse('STAGING_PATH'); } @@ -432,6 +429,8 @@ export function finalizeCandidate({ repository, staging }) { // 3. Stage maps and build metadata owner-only outside the package, then finalize the JS. mkdirSync(join(staging, 'maps'), { mode: 0o700 }); + // The creation mode passes through the umask; the staged directory keeps the exact owner-only mode. + if (MODE_EVIDENCE === 'posix') chmodSync(join(staging, 'maps'), 0o700); for (const item of prepared) writePrivate(join(staging, 'maps', `${item.jsName}.map`), item.map); writePrivate(join(staging, 'metafile-esm.json'), metafileBytes); for (const item of prepared) replaceOutput(join(dist, item.jsName), item.code, item.mode); diff --git a/tools/cli-package/finalize.test.mjs b/tools/cli-package/finalize.test.mjs index f9a2c2b3..74f6ae02 100644 --- a/tools/cli-package/finalize.test.mjs +++ b/tools/cli-package/finalize.test.mjs @@ -1,6 +1,7 @@ import assert from 'node:assert/strict'; import { chmodSync, + existsSync, mkdirSync, mkdtempSync, readFileSync, @@ -279,6 +280,11 @@ for (const [label, change, code] of [ 'MAP_SOURCE_NOT_ORIGINAL', ], ['a missing source', (map) => (map.sources = ['../src/missing.ts']), 'MAP_SOURCE_MISSING'], + [ + 'more sources than the bound', + (map) => (map.sources = Array.from({ length: 4097 }, (_, i) => `../src/s${i}.ts`)), + 'UPSTREAM_MAP_SHAPE', + ], ]) { test(`refuses an upstream map with ${label}`, async () => { const candidate = await fixture(); @@ -326,6 +332,13 @@ test('refuses final maps that embed sources, lose coverage or carry unreviewed f [(map) => (map.file = 'other.js'), 'MAP_SHAPE'], [(map) => (map.mappings = ''), 'MAP_SHAPE'], [(map) => (map.version = 2), 'MAP_SHAPE'], + [(map) => (map.mappings = 'AAAA!'), 'MAP_SHAPE'], + [(map) => (map.names = [1]), 'MAP_SHAPE'], + [ + (map) => (map.sources = Array.from({ length: 4097 }, (_, i) => `../src/s${i}.ts`)), + 'MAP_SHAPE', + ], + [(map) => (map.names = Array.from({ length: 1_000_001 }, () => '')), 'MAP_SHAPE'], [(map) => (map.sources = ['https://example.invalid/a.ts']), 'MAP_SOURCE_PATH'], ]) { const candidate = await fixture(); @@ -376,6 +389,19 @@ test('accounts for every emitted file and every consumed input', async () => { delete metafile.outputs['dist/index.js.map']; }); refusesBeforeWriting(unpaired, 'MAP_PAIRING'); + const crowded = await fixture(); + editJson(join(crowded.dist, 'metafile-esm.json'), (metafile) => { + for (let i = 0; i < 257; i += 1) { + metafile.outputs[`dist/extra${i}.js`] = { imports: [], exports: [], inputs: {}, bytes: 1 }; + } + }); + refusesBeforeWriting(crowded, 'OUTPUT_INVENTORY'); + const unsized = await fixture(); + editJson(join(unsized.dist, 'metafile-esm.json'), (metafile) => { + const contribution = Object.values(metafile.outputs['dist/index.js'].inputs)[0]; + delete contribution.bytesInOutput; + }); + refusesBeforeWriting(unsized, 'METAFILE_SHAPE'); const foreign = await fixture(); editJson(join(foreign.dist, 'metafile-esm.json'), (metafile) => { metafile.inputs['../../node_modules/left-pad/index.js'] = { @@ -463,6 +489,15 @@ test('requires an empty private staging directory outside the candidate', async (error) => error instanceof CandidateFinalizationError && error.code === 'STAGING_PATH', ); rmSync(inside, { recursive: true }); + // A child whose name merely starts with two dots is inside the candidate. + const dotted = join(candidate.root, '..private'); + mkdirSync(dotted, { mode: 0o700 }); + chmodSync(dotted, 0o700); + assert.throws( + () => finalizeCandidate({ repository: candidate.root, staging: dotted }), + (error) => error instanceof CandidateFinalizationError && error.code === 'STAGING_PATH', + ); + rmSync(dotted, { recursive: true }); // Staging must be named canonically even outside the candidate. const outside = join(scratch, `outside-staging-${count}`); mkdirSync(outside, { mode: 0o700 }); @@ -526,3 +561,55 @@ test('ordinary package and CLI builds stay map-free outside a named candidate', assert.deepEqual(config.match(/^\s*sourcemap:.*$/gmu), [' sourcemap: CANDIDATE,']); assert.match(config, /if \(CANDIDATE\) options\.sourcesContent = false;/u); }); + +test('keeps exact staged and public modes whatever the umask', { skip: !posixModes }, async () => { + const candidate = await fixture(); + // A umask that removes the owner's write bit would leave every file read-only without exact modes. + const previous = process.umask(0o277); + try { + finalizeCandidate({ repository: candidate.root, staging: candidate.staging }); + } finally { + process.umask(previous); + } + assert.equal(statSync(join(candidate.staging, 'maps')).mode & 0o777, 0o700); + for (const name of ['metafile-esm.json', 'finalized.json']) { + assert.equal(statSync(join(candidate.staging, name)).mode & 0o777, 0o600, name); + } + for (const name of readdirSync(join(candidate.staging, 'maps'))) { + assert.equal(statSync(join(candidate.staging, 'maps', name)).mode & 0o777, 0o600, name); + } + for (const name of Object.keys(candidate.emitted)) { + assert.equal( + statSync(join(candidate.dist, name)).mode & 0o777, + name === 'index.js' ? 0o755 : 0o644, + name, + ); + } +}); + +test('compares staging and the candidate by the letter case the filesystem stores', async (t) => { + const candidate = await fixture(); + const parent = dirname(candidate.root); + const variant = join(parent, basename(candidate.root).toUpperCase()); + if (!existsSync(variant)) { + t.skip('case-sensitive filesystem'); + return; + } + const inside = join(candidate.cli, 'private'); + mkdirSync(inside, { mode: 0o700 }); + chmodSync(inside, 0o700); + // Another letter case of a path inside the candidate is not its canonical name. + assert.throws( + () => + finalizeCandidate({ + repository: candidate.root, + staging: join(variant, 'apps', 'cli', 'private'), + }), + (error) => error instanceof CandidateFinalizationError && error.code === 'STAGING_PATH', + ); + // Nor does another letter case of the repository hide the canonical staging inside it. + assert.throws( + () => finalizeCandidate({ repository: variant, staging: inside }), + (error) => error instanceof CandidateFinalizationError && error.code === 'STAGING_PATH', + ); +}); diff --git a/tools/cli-package/paths.mjs b/tools/cli-package/paths.mjs new file mode 100644 index 00000000..4c5d45f0 --- /dev/null +++ b/tools/cli-package/paths.mjs @@ -0,0 +1,12 @@ +/** Path containment shared by the producer, the candidate tree and the finalizer. */ +import { isAbsolute, relative, sep } from 'node:path'; + +/** + * Whether `child` is `parent` or lies inside it, decided by path segments: a child whose first segment + * merely starts with `..` (`..private`, `...`) is inside, while `..` and `../x` are not. + * @param {string} parent @param {string} child + */ +export function within(parent, child) { + const path = relative(parent, child); + return path === '' || (path !== '..' && !path.startsWith(`..${sep}`) && !isAbsolute(path)); +} diff --git a/tools/cli-package/prepack.test.mjs b/tools/cli-package/prepack.test.mjs index 0bc3eabe..95c159ed 100644 --- a/tools/cli-package/prepack.test.mjs +++ b/tools/cli-package/prepack.test.mjs @@ -31,6 +31,7 @@ function fixture({ buildExit = 0 } = {}) { 'cli-package/prepack.mjs', 'cli-package/finalize.mjs', 'cli-package/files.mjs', + 'cli-package/paths.mjs', 'bundle-closure/closure.mjs', 'bundle-closure/inventory.mjs', 'bundle-closure/check.mjs', diff --git a/tools/cli-package/process.mjs b/tools/cli-package/process.mjs index 8cee4d14..232b20d7 100644 --- a/tools/cli-package/process.mjs +++ b/tools/cli-package/process.mjs @@ -93,6 +93,39 @@ export function stopLiveCommands() { liveChildren.clear(); } +/** The interrupts a tool forwards to the children that own their cleanup. */ +const FORWARDED_INTERRUPTS = Object.freeze(['SIGINT', 'SIGTERM', 'SIGHUP']); + +/** + * While a tool waits on children that clean up after themselves (the producer), forward SIGINT, + * SIGTERM and SIGHUP to every child tree `runCommand` owns instead of dying first: a detached child + * shares no terminal process group, so it would never receive the interrupt. On Windows the children + * share the console and receive Ctrl-C themselves. The caller runs its own cleanup, calls `dispose` + * and exits with 128 + the number of the signal `received` reports. + * @returns {{received: () => string | undefined, dispose: () => void}} + */ +export function forwardInterrupts() { + let received; + const forward = (signal) => { + received ??= signal; + for (const child of liveChildren) { + if (!child.grouped) continue; + try { + process.kill(-child.pid, signal); + } catch { + // The group already exited. + } + } + }; + for (const signal of FORWARDED_INTERRUPTS) process.on(signal, forward); + return { + received: () => received, + dispose: () => { + for (const signal of FORWARDED_INTERRUPTS) process.off(signal, forward); + }, + }; +} + export function runCommand(command, args, options = {}) { const { timeoutMs = 300_000, maxBytes = 8 * 1024 * 1024, ...spawnOptions } = options; assert.ok(Number.isSafeInteger(timeoutMs) && timeoutMs > 0); diff --git a/tools/cli-package/process.test.mjs b/tools/cli-package/process.test.mjs index 318db0f3..cf9922a6 100644 --- a/tools/cli-package/process.test.mjs +++ b/tools/cli-package/process.test.mjs @@ -1,6 +1,16 @@ import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; import { test } from 'node:test'; -import { mkdtempSync, mkdirSync, writeFileSync, rmSync, realpathSync } from 'node:fs'; +import { setTimeout as delay } from 'node:timers/promises'; +import { + existsSync, + mkdtempSync, + mkdirSync, + readFileSync, + writeFileSync, + rmSync, + realpathSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { @@ -155,3 +165,74 @@ test('actual pnpm install command shim runs help/version and a broken or absent rmSync(root, { recursive: true, force: true }); } }); + +function alive(pid) { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +async function waitForFile(file) { + for (let waited = 0; !existsSync(file) || readFileSync(file, 'utf8') === ''; waited += 25) { + assert.ok(waited < 20_000, `${file} never appeared`); + await delay(25); + } + return readFileSync(file, 'utf8'); +} + +test( + 'a tool forwards an interrupt to the detached children that clean up after themselves', + { skip: process.platform === 'win32' }, + async () => { + const root = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-forwarded-interrupt-'))); + const pidFile = join(root, 'child.pid'); + const marker = join(root, 'received'); + // The child cleans up on SIGINT, as the producer does; a SIGKILL would leave no marker. + const childCode = + `const { writeFileSync } = require('node:fs');` + + `process.on('SIGINT', () => { writeFileSync(${JSON.stringify(marker)}, 'SIGINT'); process.exit(0); });` + + `writeFileSync(${JSON.stringify(pidFile)}, String(process.pid));` + + `setInterval(() => {}, 1000);`; + const harness = ` +import { constants } from 'node:os'; +import { forwardInterrupts, isolatedEnvironment, runCommand } from ${JSON.stringify(new URL('./process.mjs', import.meta.url).href)}; +const interrupts = forwardInterrupts(); +try { + await runCommand(process.execPath, ['-e', ${JSON.stringify(childCode)}], { + env: isolatedEnvironment(${JSON.stringify(root)}), + timeoutMs: 60_000, + }); +} catch (error) { + if (interrupts.received() === undefined) throw error; +} finally { + interrupts.dispose(); +} +const signal = interrupts.received(); +if (signal !== undefined) process.exit(128 + constants.signals[signal]); +`; + const tool = spawn(process.execPath, ['--input-type=module', '-e', harness], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + let child; + try { + child = Number(await waitForFile(pidFile)); + assert.ok(alive(child)); + const exited = new Promise((resolve) => tool.once('exit', resolve)); + tool.kill('SIGINT'); + const bound = delay(20_000).then(() => 'still running'); + assert.equal(await Promise.race([exited, bound]), 130); + assert.equal(readFileSync(marker, 'utf8'), 'SIGINT'); + for (let waited = 0; alive(child); waited += 25) { + assert.ok(waited < 5_000, 'the child outlived the forwarded interrupt'); + await delay(25); + } + } finally { + if (tool.exitCode === null && tool.signalCode === null) tool.kill('SIGKILL'); + if (child !== undefined && alive(child)) process.kill(child, 'SIGKILL'); + rmSync(root, { recursive: true, force: true }); + } + }, +); diff --git a/tools/cli-package/produce.mjs b/tools/cli-package/produce.mjs index 46f7ea35..3a73b4b1 100644 --- a/tools/cli-package/produce.mjs +++ b/tools/cli-package/produce.mjs @@ -11,8 +11,9 @@ import assert from 'node:assert/strict'; import { chmodSync, mkdirSync, mkdtempSync, readdirSync, realpathSync, rmSync } from 'node:fs'; import { constants, tmpdir } from 'node:os'; -import { isAbsolute, join, relative } from 'node:path'; +import { join } from 'node:path'; import { assertNoBuildCache, globalExcludesFile, prepareCandidate } from './candidate.mjs'; +import { within } from './paths.mjs'; import { isolatedEnvironment, runPnpm, stopLiveCommands } from './process.mjs'; import { sealPackage } from './seal.mjs'; import { emitFreshPackage } from './pipeline.mjs'; @@ -20,12 +21,6 @@ import { emitFreshPackage } from './pipeline.mjs'; /** The interrupts after which the producer still removes everything private before it exits. */ const INTERRUPTS = Object.freeze(['SIGINT', 'SIGTERM', 'SIGHUP']); -/** @param {string} parent @param {string} child */ -function within(parent, child) { - const path = relative(parent, child); - return path === '' || (!path.startsWith('..') && !isAbsolute(path)); -} - /** * Produce one package-test artifact from the original checkout `repository` into the empty * `destination`. The candidate tree and its private staging are siblings in one owner-only temporary @@ -36,7 +31,8 @@ function within(parent, child) { * @param {{repository: string, destination: string, environment: Record, * temporary?: string, ports?: {prepare?: typeof prepareCandidate, emit?: typeof emitFreshPackage, * seal?: typeof sealPackage, noBuildCache?: typeof assertNoBuildCache, - * excludesFile?: typeof globalExcludesFile, pnpm?: typeof runPnpm}}} options + * excludesFile?: typeof globalExcludesFile, pnpm?: typeof runPnpm, + * remove?: (path: string) => void}}} options */ export async function producePackage({ repository, @@ -52,14 +48,15 @@ export async function producePackage({ noBuildCache = assertNoBuildCache, excludesFile = globalExcludesFile, pnpm = runPnpm, + remove = (path) => rmSync(path, { recursive: true, force: true }), } = ports; mkdirSync(destination, { recursive: true }); assert.deepEqual(readdirSync(destination), [], 'PACKED_CLI_FRESH_DESTINATION_REQUIRED'); - const home = realpathSync(mkdtempSync(join(temporary, 'relavium-package-build-'))); + const home = realpathSync.native(mkdtempSync(join(temporary, 'relavium-package-build-'))); let root; const discard = () => { - if (root !== undefined) rmSync(root, { recursive: true, force: true }); - rmSync(home, { recursive: true, force: true }); + if (root !== undefined) remove(root); + remove(home); }; const interrupt = (signal) => { stopLiveCommands(); @@ -71,16 +68,18 @@ export async function producePackage({ }; for (const signal of INTERRUPTS) process.on(signal, interrupt); try { - root = realpathSync(mkdtempSync(join(temporary, 'relavium-package-candidate-'))); + root = realpathSync.native(mkdtempSync(join(temporary, 'relavium-package-candidate-'))); chmodSync(root, 0o700); const candidate = join(root, 'source'); const staging = join(root, 'staging'); mkdirSync(candidate, { mode: 0o700 }); mkdirSync(staging, { mode: 0o700 }); chmodSync(staging, 0o700); - const published = realpathSync(destination); + // Native real paths carry the letter case the filesystem stores, so no alias passes for elsewhere. + const published = realpathSync.native(destination); + const checkout = realpathSync.native(repository); assert.ok( - !within(root, published) && !within(published, root) && !within(repository, root), + !within(root, published) && !within(published, root) && !within(checkout, root), 'PACKED_CLI_PRIVATE_STAGING_PLACEMENT', ); const env = isolatedEnvironment(home); @@ -111,7 +110,9 @@ export async function producePackage({ }); return { receipt, identity }; } finally { - for (const signal of INTERRUPTS) process.off(signal, interrupt); + // The handlers stay until everything private is gone: a late interrupt waits for the removal + // instead of ending the process in the middle of it. discard(); + for (const signal of INTERRUPTS) process.off(signal, interrupt); } } diff --git a/tools/cli-package/produce.test.mjs b/tools/cli-package/produce.test.mjs index 274eb52d..ad38c3e2 100644 --- a/tools/cli-package/produce.test.mjs +++ b/tools/cli-package/produce.test.mjs @@ -12,7 +12,7 @@ import { writeFileSync, } from 'node:fs'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { basename, dirname, join } from 'node:path'; import { after, test } from 'node:test'; import { setTimeout as delay } from 'node:timers/promises'; import { assertNoBuildCache } from './candidate.mjs'; @@ -79,11 +79,15 @@ function leftovers(temporary) { return readdirSync(temporary).filter((name) => name.startsWith('relavium-package-')); } +const INTERRUPTS = ['SIGINT', 'SIGTERM', 'SIGHUP']; +const listeners = () => INTERRUPTS.map((signal) => process.listenerCount(signal)); + test('the producer builds and seals only the owned candidate, never the original checkout', async () => { const repository = fresh('original-checkout'); const destination = join(fresh('artifact-parent'), 'artifact'); const temporary = fresh('tmp'); const { seen, ports: fakes } = ports(); + const before = listeners(); const result = await producePackage({ repository, destination, @@ -115,6 +119,8 @@ test('the producer builds and seals only the owned candidate, never the original } // The private root and the build home are gone, and only the tarball remains in the artifact. assert.deepEqual(leftovers(temporary), []); + // The interrupt handlers are gone too. + assert.deepEqual(listeners(), before); assert.equal(existsSync(candidate), false); assert.deepEqual(readdirSync(destination), ['relavium-0.1.1.tgz']); }); @@ -139,6 +145,7 @@ test('a build cache left in the candidate refuses before the seal, and nothing p test('a failure inside the build still removes the private root', async () => { const temporary = fresh('tmp'); const { ports: fakes } = ports({ failEmit: true }); + const before = listeners(); await assert.rejects( producePackage({ repository: fresh('original-checkout'), @@ -150,6 +157,7 @@ test('a failure inside the build still removes the private root', async () => { /synthetic emit failure/u, ); assert.deepEqual(leftovers(temporary), []); + assert.deepEqual(listeners(), before); }); test('private staging never shares a path with the checkout or the artifact', async () => { @@ -170,6 +178,35 @@ test('private staging never shares a path with the checkout or the artifact', as ); assert.equal(seen.prepare, undefined); assert.deepEqual(leftovers(inside), []); + // A directory whose name merely starts with two dots is inside the checkout too. + const dotted = join(repository, '..tmp'); + mkdirSync(dotted); + await assert.rejects( + producePackage({ + repository, + destination: join(fresh('artifact-parent'), 'artifact'), + environment: {}, + temporary: dotted, + ports: ports().ports, + }), + /PRIVATE_STAGING_PLACEMENT/u, + ); + assert.deepEqual(leftovers(dotted), []); + // On a case-insensitive filesystem, another letter case of the checkout is the checkout. + const variant = join(dirname(repository), basename(repository).toUpperCase()); + if (existsSync(variant)) { + await assert.rejects( + producePackage({ + repository, + destination: join(fresh('artifact-parent'), 'artifact'), + environment: {}, + temporary: join(variant, 'tmp'), + ports: ports().ports, + }), + /PRIVATE_STAGING_PLACEMENT/u, + ); + assert.deepEqual(leftovers(inside), []); + } // So would a temporary directory inside the artifact directory. const artifact = join(fresh('artifact-parent'), 'artifact'); mkdirSync(artifact); @@ -198,7 +235,7 @@ test('private staging never shares a path with the checkout or the artifact', as }); /** A producer run whose build is a long-lived child, as `pnpm pack` is, behind fake ports. */ -function interruptedProducer(temporary, destination, pidFile) { +function interruptedProducer(temporary, destination, pidFile, grandchildFile) { const url = (name) => JSON.stringify(new URL(name, import.meta.url).href); const script = ` import { writeFileSync } from 'node:fs'; @@ -218,7 +255,14 @@ await producePackage({ writeFileSync(join(staging, 'index.js.map'), 'private map'); writeFileSync(join(destination, 'relavium-0.1.1.tgz'), 'unsealed'); await runCommand(process.execPath, ['-e', ${JSON.stringify( - `require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, String(process.pid)); setInterval(() => {}, 1000);`, + `const { spawn } = require('node:child_process');` + + `const { writeFileSync } = require('node:fs');` + + // A grandchild in the same process group, as tsup is under pnpm pack and prepack. + `spawn(process.execPath, ['-e', ${JSON.stringify( + `require('node:fs').writeFileSync(${JSON.stringify(grandchildFile)}, String(process.pid)); setInterval(() => {}, 1000);`, + )}], { stdio: 'ignore' });` + + `writeFileSync(${JSON.stringify(pidFile)}, String(process.pid));` + + `setInterval(() => {}, 1000);`, )}], { env: isolatedEnvironment(${JSON.stringify(temporary)}), timeoutMs: 60_000 }); }, }, @@ -241,6 +285,7 @@ function alive(pid) { for (const [signal, code] of [ ['SIGINT', 130], ['SIGTERM', 143], + ['SIGHUP', 129], ]) { test( `an interrupted producer (${signal}) stops its build and removes everything private`, @@ -249,33 +294,79 @@ for (const [signal, code] of [ const temporary = fresh('tmp'); mkdirSync(join(temporary, 'checkout')); const destination = join(fresh('artifact-parent'), 'artifact'); - const pidFile = join(fresh('pid'), 'sleeper.pid'); - const child = interruptedProducer(temporary, destination, pidFile); - let sleeper; + const pids = fresh('pid'); + const pidFile = join(pids, 'sleeper.pid'); + const grandchildFile = join(pids, 'grandchild.pid'); + const child = interruptedProducer(temporary, destination, pidFile, grandchildFile); + const tree = []; try { - for ( - let waited = 0; - !existsSync(pidFile) || readFileSync(pidFile, 'utf8') === ''; - waited += 25 - ) { - assert.ok(waited < 20_000, 'the fake build never started'); - await delay(25); + for (const file of [pidFile, grandchildFile]) { + for ( + let waited = 0; + !existsSync(file) || readFileSync(file, 'utf8') === ''; + waited += 25 + ) { + assert.ok(waited < 20_000, 'the fake build never started'); + await delay(25); + } + tree.push(Number(readFileSync(file, 'utf8'))); } - sleeper = Number(readFileSync(pidFile, 'utf8')); - assert.ok(alive(sleeper)); + assert.ok(tree.every(alive)); const exited = new Promise((resolve) => child.once('exit', resolve)); child.kill(signal); assert.equal(await exited, code); - for (let waited = 0; alive(sleeper); waited += 25) { - assert.ok(waited < 5_000, 'the build child outlived the interrupt'); + // The whole build tree is stopped, not only the process the producer started. + for (let waited = 0; tree.some(alive); waited += 25) { + assert.ok(waited < 5_000, 'part of the build tree outlived the interrupt'); await delay(25); } assert.deepEqual(leftovers(temporary), []); assert.deepEqual(readdirSync(destination), []); } finally { if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); - if (sleeper !== undefined && alive(sleeper)) process.kill(sleeper, 'SIGKILL'); + for (const pid of tree) if (alive(pid)) process.kill(pid, 'SIGKILL'); } }, ); } + +test( + 'an interrupt during the final removal waits for it instead of ending the process', + { skip: !posixModes }, + async () => { + const temporary = fresh('tmp'); + mkdirSync(join(temporary, 'checkout')); + const destination = join(fresh('artifact-parent'), 'artifact'); + const url = (name) => JSON.stringify(new URL(name, import.meta.url).href); + // The private root's removal itself is interrupted; the handlers must still be installed then. + const script = ` +import { rmSync } from 'node:fs'; +import { producePackage } from ${url('./produce.mjs')}; +await producePackage({ + repository: ${JSON.stringify(join(temporary, 'checkout'))}, + destination: ${JSON.stringify(destination)}, + environment: {}, + temporary: ${JSON.stringify(temporary)}, + ports: { + pnpm: async () => ({ stdout: '9.12.3\\n', stderr: '' }), + excludesFile: async () => undefined, + prepare: async () => ({ source: '${SHA}', dirty: false, files: 1 }), + emit: async ({ staging }) => ({ staging }), + remove: (path) => { + if (path.includes('relavium-package-candidate-')) process.kill(process.pid, 'SIGINT'); + rmSync(path, { recursive: true, force: true }); + }, + }, +}); +`; + const child = spawn(process.execPath, ['--input-type=module', '-e', script], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + const [code, signal] = await new Promise((resolve) => + child.once('exit', (exitCode, signalCode) => resolve([exitCode, signalCode])), + ); + assert.equal(signal, null); + assert.equal(code, 0); + assert.deepEqual(leftovers(temporary), []); + }, +); From aa5c994a93b960b44220c006d6b77816b17c6ce7 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Thu, 22 Oct 2026 09:56:41 +0300 Subject: [PATCH 25/97] fix(repo): address the third Step 9a.1 review round Round 3 (at 184562bd) reported eleven verified findings, two medium; its security lens was clean. - An interrupt that arrived after the build's last child exited, during the seal or the final removal, was dropped: Node delivers a caught signal only when the event loop next polls, and the handlers were removed first. Round 2's test asserted exit 0 and so pinned the dropped interrupt. exitOnInterrupt (process.mjs) now keeps the handlers until the loop has polled once more. That takes two turns when the work ran from an I/O callback; one turn was measured to drop the signal. The producer then empties the artifact directory and exits with the signal's code, tested from an I/O-callback tail. - An interrupted consumer stops its children, the bootstrap witness's included (tracked in tools/cli-bootstrap), removes its installation and fixtures and exits with the signal's code; completeConsumer requires that abandon step. - pnpm smoke:cli-package starts no later stage once an interrupt arrived, removes its root with retries, and is tested end to end from a copied tooling tree (check.test.mjs). - The producer's isolated build environment, its native real path of the checkout and the prepack catch-all gained killing tests. - Git reading the original checkout uses no home, global or system configuration (isolatedGitEnvironment); a skip-worktree or assume-unchanged file makes the source identity dirty. - An upstream map may map nothing only for tsc's type-only `export {};` output. - packaging.md, the README, current.md and the 9a.1 checkpoint record the corrections and round 3. Gates: lint:tools, typecheck:tools, every pnpm run ci step except the filesystem and secrets families (their captured sources are unchanged), with the real producer in smoke:cli-package, and the acceptance tools under CI=true GITHUB_ACTIONS=true; a mutation run of thirty-one mutants over the corrections kills all of them. Refs: ADR-0111, ADR-0051 Co-Authored-By: Claude Opus 5.5 --- docs/reference/cli/packaging.md | 24 +- docs/roadmap/current.md | 2 +- .../phase-2.5.5-hardening-and-remediation.md | 1 + package.json | 2 +- tools/cli-bootstrap/child.mjs | 18 +- tools/cli-bootstrap/process.mjs | 25 ++ tools/cli-bootstrap/witness.test.mjs | 94 ++++++- tools/cli-package/README.md | 14 +- tools/cli-package/candidate.mjs | 37 ++- tools/cli-package/candidate.test.mjs | 99 +++++++- tools/cli-package/check.mjs | 17 +- tools/cli-package/check.test.mjs | 230 ++++++++++++++++++ tools/cli-package/consume.mjs | 16 +- tools/cli-package/consumer-lifetime.mjs | 64 +++-- tools/cli-package/consumer-lifetime.test.mjs | 122 ++++++++++ tools/cli-package/finalize.mjs | 7 +- tools/cli-package/finalize.test.mjs | 2 + tools/cli-package/prepack.test.mjs | 20 +- tools/cli-package/process.mjs | 73 +++++- tools/cli-package/process.test.mjs | 52 +++- tools/cli-package/produce.mjs | 119 +++++---- tools/cli-package/produce.test.mjs | 128 ++++++++-- 22 files changed, 1002 insertions(+), 164 deletions(-) create mode 100644 tools/cli-package/check.test.mjs diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index e7f5e995..2410f25a 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -236,10 +236,13 @@ describes the implemented mechanism; independent acceptance is recorded in the [Wave 3 register](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments). - **Owned candidate tree.** `pnpm package:cli:pack` never builds in the developer's - workspace. It records the original checkout's commit and full porcelain status, copies - every file Git would ship (tracked plus untracked-but-not-ignored, minus deletions, by - the repository's ignore rules and the developer's global excludes file) into a fresh - owner-only tree, and refuses if that commit or status changed during the copy. + workspace. It records the original checkout's commit, full porcelain status and the + tracked files Git is told not to check (skip-worktree or assume-unchanged, which make + the identity `dirty` whatever their bytes), copies every file Git would ship (tracked + plus untracked-but-not-ignored, minus deletions, by the repository's ignore rules and + the developer's global excludes file, passed explicitly; that Git reads no home, global + or system configuration) into a fresh owner-only tree, and refuses if any of the three + changed during the copy. Ignored output, caches, dependencies and private notes never enter it. The locked closure is installed there offline, frozen and with `--ignore-scripts` from the existing content-addressed store, discovered with pnpm's store-location variables @@ -263,7 +266,8 @@ describes the implemented mechanism; independent acceptance is recorded in the output paired with exactly one map, plus the metafile, nothing else). It validates every consumed upstream tsc map: exact terminal map link, version 3, matching `file`, empty `sourceRoot`, no embedded sources, and only that package's existing original - TypeScript (a type-only module may have empty mappings). It validates every final map: + TypeScript; its mappings may be empty only when the module's types all erase, which tsc + emits as exactly `export {};`. It validates every final map: version 3, no embedded sources or unreviewed fields, and sources resolving to existing `apps/cli/src` or engine `src` TypeScript that equal exactly the original sources of the inputs contributing bytes to that output, so a bundler fallback to compiled JS @@ -285,9 +289,13 @@ describes the implemented mechanism; independent acceptance is recorded in the Private maps, the metafile, the hash record and the candidate tree are removed when the producer succeeds, fails or is interrupted (SIGINT, SIGTERM, SIGHUP); an interrupt also stops the build's child processes and empties the artifact directory, which was empty at the start, -and the local `pnpm smoke:cli-package` forwards an interrupt to the producer it started rather than -ending first. A producer killed outright, a crash or a lost host can leave the owner-only root in the -temporary directory, and no secure erasure is claimed. The private map manifest, actual packed CLI and engine +before the producer exits with the signal's code — also when it arrives after the build's last +child exited, during the seal or the final removal. An interrupted consumer likewise stops its +child processes, including the bootstrap witness's, and removes its installation and fixtures. +The local `pnpm smoke:cli-package` forwards an interrupt to the producer or consumer it started +rather than ending first, starts no later stage once one arrived, and exits with the signal's +code after removing its root. A producer killed outright, a crash or a lost host can leave the +owner-only root in the temporary directory, and no secure erasure is claimed. The private map manifest, actual packed CLI and engine symbolication, `check:map-controls` and the `package_test` receipt flip are Step 9a.2; recipient custody, the release state controller and the workflow graph are Steps 9b–9d. Until those and Wave 4b's production recipient exist, the release workflow's publish job diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index ee794f6b..4224a310 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -171,7 +171,7 @@ bar and round cap, as its medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, by the author alone). Its other findings are recorded in -[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is implemented on `development` and under review ([checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-implementation-checkpoint--2026-10-22)); its +[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is implemented on `development` and under review ([checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-implementation-checkpoint--2026-10-22)); the substrate's 9a.1–9d increments carry a dedicated archive/crypto security sitting. Steps 10–19 follow the [increment table](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) and end with whole-wave acceptance at Step 19. `development` is pushed after each accepted increment; PR #95 no longer bounds the diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 6be8f0aa..de178d6d 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -9580,6 +9580,7 @@ adversarially verified; the detailed reports stay local and Git-ignored. | --- | --- | --- | --- | --- | | 1 | `a72c7ecd` | NONCLEAN (plan lens clean) | 9 (2 medium, 6 low, 1 nit) | an interrupted producer (SIGINT, SIGTERM, SIGHUP) left its private maps and candidate tree in the temporary directory while its detached build ran on, so the producer now stops the child trees `runCommand` tracks and removes the private root, the build home and the artifact directory's contents before exiting with the signal's code (a producer killed outright is a named limit); private staging must be canonical and outside the candidate by real path; the global-excludes wiring, the producer environment, the placement check's artifact half, the candidate's bounds and three finalizer refusals gained killing tests; the status lines, the CI pack-step comment and the post-closure CI record were corrected | | 2 | `8ddb7ec2` | NONCLEAN (plan and integration lenses clean) | 12 (4 medium, 7 low, 1 nit; 9 distinct) | the producer removed its interrupt handlers before the final removal, so an interrupt during it ended the process with the private root half removed: the handlers now stay until everything private is gone, and a deterministic test interrupts that removal; SIGHUP, the handlers' removal and a whole build tree (a grandchild in the build's process group) are tested; path containment is decided by segments in one shared helper (`..private` is inside); private staging and the producer's placement compare the real paths and letter case the filesystem stores (`realpathSync.native`); the local `pnpm smoke:cli-package` forwards an interrupt to the producer and the consumer it started (`forwardInterrupts`); the finalizer creates its maps directory with an exact mode and its staged and public modes are tested under a umask that removes the owner's write bit; further finalizer and staged-closure refusals gained killing tests; the ADR-0051 note links ADR-0111 | +| 3 | `184562bd` | NONCLEAN (security lens clean) | 11 (2 medium, 8 low, 1 nit) | round 2 still dropped an interrupt that arrived after the build's last child exited, during the seal or the final removal, because Node delivers a caught signal only when the event loop next polls and the handlers were removed first; round 2's test asserted exit 0 and so pinned the dropped interrupt. Every tool now keeps its handlers until the loop has polled once more, which takes two turns when the work ran from an I/O callback (one turn, the suggested fix, was measured to drop it), and the producer then empties the artifact directory and exits with the signal's code, tested from an I/O-callback tail. An interrupted consumer stops its own and the bootstrap witness's children and removes its installation and fixtures, and the local `pnpm smoke:cli-package`, now tested end to end, starts no later stage once an interrupt arrived. The build's isolated environment, the producer's native real path of the checkout and the prepack catch-all gained killing tests. Git reading the checkout uses no home, global or system configuration, a skip-worktree or assume-unchanged file makes the identity dirty, and an empty upstream mapping is accepted only for `export {};` | **Remaining gates.** The increment is not independently accepted. Two NEW consecutive clean complete cumulative rounds after the last material fix and a separate security sitting remain (the Acceptance diff --git a/package.json b/package.json index 569e6b9c..0d6dcf71 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "smoke:budget-replay": "node tools/budget-replay-compat/check.mjs", "check:wire-mutations": "node tools/wire-conformance/check.mjs", "check:usage-mutations": "node tools/usage-coverage-controls/check.mjs", - "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", + "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/check.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", "check:fs-mutations": "node tools/fs-policy-controls/check.mjs", "check:secrets-mutations": "node tools/secrets-controls/check.mjs", "smoke:cli-bootstrap": "node tools/cli-bootstrap/check.mjs", diff --git a/tools/cli-bootstrap/child.mjs b/tools/cli-bootstrap/child.mjs index 66cb2703..4207414d 100644 --- a/tools/cli-bootstrap/child.mjs +++ b/tools/cli-bootstrap/child.mjs @@ -13,7 +13,7 @@ import { tmpdir } from 'node:os'; import { join, relative } from 'node:path'; import { pathToFileURL } from 'node:url'; import { createHash } from 'node:crypto'; -import { spawnBounded } from './process.mjs'; +import { spawnBounded, stopBoundedChildren } from './process.mjs'; import { maxWitnessBytes } from './policy.mjs'; function snapshot(root) { @@ -34,9 +34,24 @@ function snapshot(root) { return entries; } +/** Every fixture root a running control still owns. */ +const liveFixtures = new Set(); + +/** + * For a caller that an interrupt ends before `withFixture` can remove its own root: synchronously kill + * every bounded child, then remove every fixture root still in use. + */ +export function abandonFixtures() { + stopBoundedChildren(); + // Retries absorb a killed child that was still finishing a write. + for (const root of liveFixtures) rmSync(root, { recursive: true, force: true, maxRetries: 3 }); + liveFixtures.clear(); +} + /** Each child receives a synthetic HOME/cwd, no ambient credentials or Node preload. */ export async function withFixture(run) { const root = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-bootstrap-'))); + liveFixtures.add(root); try { const home = join(root, 'home'), workspace = join(root, 'workspace'), @@ -118,6 +133,7 @@ export async function withFixture(run) { return result; } finally { // This exact mkdtemp-owned directory is the sole cleanup target, including failing controls. + liveFixtures.delete(root); rmSync(root, { recursive: true, force: true }); } } diff --git a/tools/cli-bootstrap/process.mjs b/tools/cli-bootstrap/process.mjs index 4e6e23fa..23c7d0cf 100644 --- a/tools/cli-bootstrap/process.mjs +++ b/tools/cli-bootstrap/process.mjs @@ -3,6 +3,28 @@ import { Buffer } from 'node:buffer'; import { setTimeout, clearTimeout } from 'node:timers'; export const maxOutputBytes = 256 * 1024; + +/** Every bounded child still running, so that an interrupted caller can stop them before it exits. */ +const liveChildren = new Set(); + +/** + * Synchronously kill every bounded child that has not closed yet: its process group, or the child + * itself where it has none. For a signal handler that exits without waiting for `spawnBounded`. + */ +export function stopBoundedChildren() { + for (const { child, grouped } of liveChildren) { + if (grouped) { + try { + process.kill(-child.pid, 'SIGKILL'); + continue; + } catch { + // The group already exited. + } + } + child.kill('SIGKILL'); + } + liveChildren.clear(); +} /** Bound bytes and wall time, kill only this owned process group, and wait for close. */ export function spawnBounded( args, @@ -16,6 +38,8 @@ export function spawnBounded( detached: grouped, stdio: ['ignore', 'pipe', 'pipe'], }); + const live = { child, grouped }; + if (child.pid !== undefined) liveChildren.add(live); const output = { stdout: [], stderr: [] }; let bytes = 0, timedOut = false, @@ -48,6 +72,7 @@ export function spawnBounded( spawnError = error.message; }); child.once('close', (code, signal) => { + liveChildren.delete(live); clearTimeout(timer); kill(); resolve({ diff --git a/tools/cli-bootstrap/witness.test.mjs b/tools/cli-bootstrap/witness.test.mjs index e859489e..4b75bd53 100644 --- a/tools/cli-bootstrap/witness.test.mjs +++ b/tools/cli-bootstrap/witness.test.mjs @@ -1,6 +1,18 @@ import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; import { test } from 'node:test'; -import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { setTimeout as delay } from 'node:timers/promises'; +import { + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + realpathSync, + rmSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import { join, relative } from 'node:path'; import { pathToFileURL } from 'node:url'; @@ -400,3 +412,83 @@ test('output cap counts UTF-8 bytes and joins the killed child', async () => { assert.equal(result.joined, true); assert.equal(result.signal, 'SIGKILL'); }); +test( + 'an interrupted caller abandons a running control: its bounded child is killed, its fixture removed', + { skip: process.platform === 'win32' }, + async (t) => { + const temporary = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-bootstrap-abandon-'))); + t.after(() => rmSync(temporary, { recursive: true, force: true })); + const pidFile = join(temporary, 'child.pid'); + const url = (name) => JSON.stringify(new URL(name, import.meta.url).href); + // The bounded child keeps writing into its fixture, as a running control does, and outlives the + // fixture's removal unless it is killed. + const writer = + `const { writeFileSync } = require('node:fs');` + + `const { join } = require('node:path');` + + `writeFileSync(${JSON.stringify(pidFile)}, String(process.pid));` + + `let n = 0; setInterval(() => { try { writeFileSync(join(process.env.TMPDIR, 'f' + n++), 'x'); } catch {} }, 5);`; + const script = ` +import { abandonFixtures, withFixture } from ${url('./child.mjs')}; +import { spawnBounded } from ${url('./process.mjs')}; +process.on('SIGINT', () => { + abandonFixtures(); + process.exit(130); +}); +await withFixture((fixture) => + spawnBounded(['-e', ${JSON.stringify(writer)}], { env: { TMPDIR: fixture.temporary } }, { + timeoutMs: 60_000, + }), +); +`; + const caller = spawn(process.execPath, ['--input-type=module', '-e', script], { + env: { PATH: process.env.PATH, TMPDIR: temporary }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let child; + try { + for ( + let waited = 0; + !existsSync(pidFile) || readFileSync(pidFile, 'utf8') === ''; + waited += 25 + ) { + assert.ok(waited < 20_000, 'the bounded child never started'); + await delay(25); + } + child = Number(readFileSync(pidFile, 'utf8')); + assert.equal( + readdirSync(temporary).filter((name) => name.startsWith('relavium-bootstrap-')).length, + 1, + ); + const exited = new Promise((resolve) => + caller.once('exit', (code, signal) => resolve([code, signal])), + ); + caller.kill('SIGINT'); + assert.deepEqual(await exited, [130, null]); + const alive = () => { + try { + process.kill(child, 0); + return true; + } catch { + return false; + } + }; + for (let waited = 0; alive(); waited += 25) { + assert.ok(waited < 5_000, 'the bounded child outlived the abandoned control'); + await delay(25); + } + assert.deepEqual( + readdirSync(temporary).filter((name) => name.startsWith('relavium-bootstrap-')), + [], + ); + } finally { + if (caller.exitCode === null && caller.signalCode === null) caller.kill('SIGKILL'); + if (child !== undefined) { + try { + process.kill(child, 'SIGKILL'); + } catch { + // Already stopped. + } + } + } + }, +); diff --git a/tools/cli-package/README.md b/tools/cli-package/README.md index bf2ebe49..88c483a7 100644 --- a/tools/cli-package/README.md +++ b/tools/cli-package/README.md @@ -17,8 +17,10 @@ pnpm smoke:cli-package The producer (`produce.mjs`, run by `pack.mjs`) never builds in the developer's workspace. It copies the files Git would ship, by the repository's ignore rules and the developer's -global excludes file, into a fresh owner-only candidate tree, refusing if the original -checkout's commit or status changes during the copy, and installs the locked closure there offline, frozen and +global excludes file (its Git reads no other home, global or system configuration), into a +fresh owner-only candidate tree, refusing if the original checkout's commit, status or set of +files Git is told not to check changes during the copy (such a skip-worktree or assume-unchanged +file makes the source `dirty`), and installs the locked closure there offline, frozen and without lifecycle scripts from the existing store. In that tree it emits the five inlined dependency packages in order with the installed compiler, upstream maps and no build cache. Prepack (`prepack.mjs`) performs the sole final CLI build in candidate @@ -39,9 +41,11 @@ inventories. This deliberately supports the pinned packer's dialect, not general The resulting public `package-test-receipt.json` records `compiled_package_test`. The candidate's maps are built, validated and then removed with the rest of the private staging, also when the producer is interrupted (SIGINT, SIGTERM, SIGHUP), which -stops the build first, and `pnpm smoke:cli-package` forwards an interrupt to the producer -and the consumer it started instead of ending before them; a producer killed outright can -leave its owner-only root in the temporary directory. The receipt does **not** certify symbolication, a map manifest, +stops the build first and empties the artifact directory, even after the last build child +exited. An interrupted consumer stops its children, the bootstrap witness's included, and +removes its installation. `pnpm smoke:cli-package` forwards an interrupt to the producer or +consumer it started instead of ending before them and starts no later stage after one; a +producer killed outright can leave its owner-only root in the temporary directory. The receipt does **not** certify symbolication, a map manifest, encrypted custody or a production release candidate (Steps 9a.2–9d). No private map, metafile or per-source inventory enters the public artifact. The release workflow's `release-guard.mjs` refuses every artifact until verified release custody exists: it diff --git a/tools/cli-package/candidate.mjs b/tools/cli-package/candidate.mjs index c8da57a1..780ab3f7 100644 --- a/tools/cli-package/candidate.mjs +++ b/tools/cli-package/candidate.mjs @@ -18,10 +18,11 @@ import { writeFileSync, } from 'node:fs'; import { dirname, isAbsolute, join, resolve } from 'node:path'; +import { isDeepStrictEqual } from 'node:util'; import { ENGINE_PACKAGES } from './finalize.mjs'; import { readOrdinaryFile } from './files.mjs'; import { within } from './paths.mjs'; -import { isolatedEnvironment, runCommand, runPnpm } from './process.mjs'; +import { isolatedEnvironment, isolatedGitEnvironment, runCommand, runPnpm } from './process.mjs'; export const CANDIDATE_LIMITS = Object.freeze({ files: 20_000, @@ -66,8 +67,8 @@ const GIT_CONFIG_VARIABLES = Object.freeze([ * already isolated this process's environment, otherwise `core.excludesFile` as the developer's Git * reads it, else Git's default `$XDG_CONFIG_HOME/git/ignore` or `$HOME/.config/git/ignore`; undefined * when no such file exists. Only that location is read from the developer's configuration: every - * candidate Git command keeps the isolated environment, so no global hook, fsmonitor or other setting - * applies. + * candidate Git command runs without global or system configuration (`isolatedGitEnvironment`), so no + * global hook, fsmonitor or other setting applies, wherever it is stored. * @param {{repository: string, environment: Record, run?: typeof runCommand}} options * @returns {Promise} */ @@ -121,13 +122,26 @@ function excludesArguments(excludes) { } /** - * The source identity of the original checkout: its exact commit and its full porcelain status, so - * a candidate copied from a modified tree records `dirty` and a status change during the copy - * refuses. Untracked files are judged with the developer's global excludes (`globalExcludesFile`). + * The tracked paths whose working-tree changes `git status` cannot see: entries marked skip-worktree + * (`S`) or assume-unchanged (a lowercase tag) in `git ls-files -v`. + * @param {string} output NUL-separated `git ls-files -v -z` entries + */ +function uncheckedPaths(output) { + return gitPaths(output) + .filter((entry) => entry[0] === 'S' || /^[a-z]$/u.test(entry[0])) + .map((entry) => entry.slice(2)); +} + +/** + * The source identity of the original checkout: its exact commit, its full porcelain status and the + * tracked files Git is told not to check, so a candidate copied from a modified tree records `dirty` + * and a change to either during the copy refuses. A skip-worktree or assume-unchanged file counts as + * dirty whatever its bytes, since its copy is never compared with the commit. Untracked files are + * judged with the developer's global excludes (`globalExcludesFile`). * @param {string} repository @param {typeof runCommand} [run] @param {string} [excludes] */ export async function sourceIdentity(repository, run = runCommand, excludes = undefined) { - const options = { cwd: repository, env: isolatedEnvironment(repository) }; + const options = { cwd: repository, env: isolatedGitEnvironment() }; const source = ( await run('git', ['--no-replace-objects', 'rev-parse', 'HEAD'], options) ).stdout.trim(); @@ -139,7 +153,8 @@ export async function sourceIdentity(repository, run = runCommand, excludes = un options, ) ).stdout; - return { source, status, dirty: status !== '' }; + const unchecked = uncheckedPaths((await run('git', ['ls-files', '-v', '-z'], options)).stdout); + return { source, status, unchecked, dirty: status !== '' || unchecked.length > 0 }; } /** @@ -156,7 +171,7 @@ export async function candidateFiles( excludes = undefined, limits = CANDIDATE_LIMITS, ) { - const options = { cwd: repository, env: isolatedEnvironment(repository) }; + const options = { cwd: repository, env: isolatedGitEnvironment() }; const listed = gitPaths( ( await run( @@ -343,7 +358,9 @@ export async function prepareCandidate({ }); const after = await sourceIdentity(repository, run, excludes); assert.ok( - after.source === identity.source && after.status === identity.status, + after.source === identity.source && + after.status === identity.status && + isDeepStrictEqual(after.unchecked, identity.unchecked), 'PACKED_CLI_SOURCE_CHANGED_DURING_COPY', ); const store = await dependencyStore({ repository, environment, home, pnpm }); diff --git a/tools/cli-package/candidate.test.mjs b/tools/cli-package/candidate.test.mjs index e8d6db20..c932e591 100644 --- a/tools/cli-package/candidate.test.mjs +++ b/tools/cli-package/candidate.test.mjs @@ -14,6 +14,7 @@ import { import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { after, test } from 'node:test'; +import { isDeepStrictEqual } from 'node:util'; import { assertNoBuildCache, candidateFiles, @@ -26,7 +27,7 @@ import { sourceIdentity, verifyCandidateLinks, } from './candidate.mjs'; -import { isolatedEnvironment, runCommand } from './process.mjs'; +import { isolatedEnvironment, isolatedGitEnvironment, runCommand } from './process.mjs'; const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-candidate-tree-'))); after(() => rmSync(scratch, { recursive: true, force: true })); @@ -72,27 +73,47 @@ test('source identity records the exact commit and the complete porcelain status const clean = git({ '--no-replace-objects rev-parse HEAD': `${SHA}\n`, 'status --porcelain=v1 -z --untracked-files=all': '', + 'ls-files -v -z': 'H a.ts\0H b.ts\0', }); assert.deepEqual(await sourceIdentity('/repository', clean.run), { source: SHA, status: '', + unchecked: [], dirty: false, }); assert.ok(clean.calls.every((call) => call.command === 'git' && call.cwd === '/repository')); - // The original checkout is queried with an isolated environment, never the caller's. + // The original checkout is queried with an isolated environment without any home, global or + // system configuration, never the caller's. assert.ok( - clean.calls.every((call) => call.env.HOME === '/repository' && !('NPM_TOKEN' in call.env)), + clean.calls.every( + (call) => isDeepStrictEqual(call.env, isolatedGitEnvironment()) && !('NPM_TOKEN' in call.env), + ), ); const modified = git({ '--no-replace-objects rev-parse HEAD': `${SHA}\n`, 'status --porcelain=v1 -z --untracked-files=all': '?? notes.txt\0', + 'ls-files -v -z': 'H a.ts\0', }); assert.equal((await sourceIdentity('/repository', modified.run)).dirty, true); + // A file Git is told not to check — skip-worktree (S) or assume-unchanged (a lowercase tag) — is + // copied unchecked, so it counts as dirty even when the status is empty. + const unchecked = git({ + '--no-replace-objects rev-parse HEAD': `${SHA}\n`, + 'status --porcelain=v1 -z --untracked-files=all': '', + 'ls-files -v -z': 'H a.ts\0S sparse.ts\0h assumed.ts\0s both.ts\0M merged.ts\0', + }); + assert.deepEqual(await sourceIdentity('/repository', unchecked.run), { + source: SHA, + status: '', + unchecked: ['sparse.ts', 'assumed.ts', 'both.ts'], + dirty: true, + }); // The developer's global excludes file judges untracked files. const excluded = git({ '--no-replace-objects rev-parse HEAD': `${SHA}\n`, '-c core.excludesFile=/home/someone/.config/git/ignore status --porcelain=v1 -z --untracked-files=all': '', + 'ls-files -v -z': '', }); assert.equal( (await sourceIdentity('/repository', excluded.run, '/home/someone/.config/git/ignore')).dirty, @@ -102,6 +123,7 @@ test('source identity records the exact commit and the complete porcelain status const malformed = git({ '--no-replace-objects rev-parse HEAD': head, 'status --porcelain=v1 -z --untracked-files=all': '', + 'ls-files -v -z': '', }); await assert.rejects(sourceIdentity('/repository', malformed.run), /SOURCE_IDENTITY/u); } @@ -113,6 +135,7 @@ test('candidate files are tracked plus unignored untracked files, minus deletion 'ls-files -z --deleted': 'gone.ts\0', }); assert.deepEqual(await candidateFiles('/repository', listing.run), ['a.ts', 'b.ts', 'new.ts']); + assert.ok(listing.calls.every((call) => isDeepStrictEqual(call.env, isolatedGitEnvironment()))); const empty = git({ 'ls-files -z --cached --others --exclude-standard': '', 'ls-files -z --deleted': '', @@ -253,6 +276,58 @@ test('a real checkout copies nothing its developer ignores globally', async () = assert.equal((await sourceIdentity(repository)).dirty, true); }); +/** A real repository with one committed file, `kept.ts`, created with Git's own configuration off. */ +async function committedRepository(name) { + const repository = fresh(name); + const inRepository = (args) => + runCommand('git', args, { cwd: repository, env: isolatedGitEnvironment(), timeoutMs: 30_000 }); + await inRepository(['init', '--quiet']); + write(join(repository, 'kept.ts'), 'export {};\n'); + await inRepository(['add', 'kept.ts']); + await inRepository([ + '-c', + 'user.name=t', + '-c', + 'user.email=t@example.invalid', + 'commit', + '--quiet', + '-m', + 'one', + ]); + return { repository, inRepository }; +} + +test("no configuration found inside the checkout acts as Git's global configuration", async () => { + const { repository } = await committedRepository('planted-configuration'); + write(join(repository, 'secret.local'), 'private\n'); + // A home-style .gitconfig and an XDG-style git/ignore at the checkout root, untracked, each of + // which would hide files if Git took the checkout for its home or configuration directory. + write(join(repository, 'hide'), 'secret.local\n.gitconfig\nhide\ngit/\n'); + write(join(repository, '.gitconfig'), `[core]\n\texcludesFile = ${join(repository, 'hide')}\n`); + write(join(repository, 'git', 'ignore'), 'secret.local\ngit/\nhide\n.gitconfig\n'); + assert.deepEqual(await candidateFiles(repository), [ + '.gitconfig', + 'git/ignore', + 'hide', + 'kept.ts', + 'secret.local', + ]); + assert.equal((await sourceIdentity(repository)).dirty, true); +}); + +for (const flag of ['--skip-worktree', '--assume-unchanged']) + test(`a tracked file marked ${flag} makes the identity dirty`, async () => { + const { repository, inRepository } = await committedRepository('unchecked'); + await inRepository(['update-index', flag, 'kept.ts']); + write(join(repository, 'kept.ts'), 'export const changed = true;\n'); + // Git's own status cannot see the change, and the copy would take the changed bytes. + assert.equal((await inRepository(['status', '--porcelain=v1'])).stdout, ''); + assert.deepEqual(await candidateFiles(repository), ['kept.ts']); + const identity = await sourceIdentity(repository); + assert.deepEqual(identity.unchecked, ['kept.ts']); + assert.equal(identity.dirty, true); + }); + test('the candidate copy is byte-exact, keeps only the executable bit and refuses links', () => { const repository = fresh('repository'); write(join(repository, 'package.json'), '{"name":"root"}\n'); @@ -480,7 +555,12 @@ test('a candidate build leaves no build cache behind', () => { }); /** A source repository plus fake git and pnpm ports; the fake install creates the workspace links. */ -function preparation({ statusAfterCopy = '', headAfterCopy = SHA, excludes } = {}) { +function preparation({ + statusAfterCopy = '', + headAfterCopy = SHA, + uncheckedAfterCopy = '', + excludes, +} = {}) { const repository = fresh('source'); write(join(repository, 'package.json'), '{}\n'); for (const name of ['shared', 'llm', 'core', 'db', 'mcp']) { @@ -496,6 +576,7 @@ function preparation({ statusAfterCopy = '', headAfterCopy = SHA, excludes } = { const events = []; let statuses = 0; let heads = 0; + let flags = 0; // With an excludes file, the listing and both status calls answer only when they carry it. const prefix = excludes === undefined ? '' : `-c core.excludesFile=${excludes} `; const run = async (command, args) => { @@ -512,6 +593,10 @@ function preparation({ statusAfterCopy = '', headAfterCopy = SHA, excludes } = { if (key === `${prefix}ls-files -z --cached --others --exclude-standard`) return { stdout: files.join('\0') }; if (key === 'ls-files -z --deleted') return { stdout: '' }; + if (key === 'ls-files -v -z') { + flags += 1; + return { stdout: `H package.json\0${flags === 1 ? '' : uncheckedAfterCopy}` }; + } throw new Error(`unexpected git ${key}`); }; const pnpm = async (args, options) => { @@ -544,10 +629,12 @@ test('candidate preparation copies, re-checks identity, installs and verifies li assert.deepEqual(ports.events, [ 'git --no-replace-objects rev-parse HEAD', 'git status --porcelain=v1 -z --untracked-files=all', + 'git ls-files -v -z', 'git ls-files -z --cached --others --exclude-standard', 'git ls-files -z --deleted', 'git --no-replace-objects rev-parse HEAD', 'git status --porcelain=v1 -z --untracked-files=all', + 'git ls-files -v -z', 'pnpm store path', `pnpm install --offline --frozen-lockfile --ignore-scripts --store-dir ${ports.store}`, ]); @@ -559,10 +646,12 @@ test('candidate preparation copies, re-checks identity, installs and verifies li }); test('a source change during the copy refuses before anything is installed', async () => { - // A changed status, or a commit that moved while the status stayed the same. + // A changed status, a commit that moved while the status stayed the same, or a file Git was told + // not to check during the copy. for (const change of [ { statusAfterCopy: ' M package.json\0' }, { headAfterCopy: 'b'.repeat(40) }, + { uncheckedAfterCopy: 'h tools/run.mjs\0' }, ]) { const ports = preparation(change); await assert.rejects( diff --git a/tools/cli-package/check.mjs b/tools/cli-package/check.mjs index c38d44a4..10f72ee0 100644 --- a/tools/cli-package/check.mjs +++ b/tools/cli-package/check.mjs @@ -10,10 +10,13 @@ import { forwardInterrupts, isolatedEnvironment, runCommand } from './process.mj assert.equal(process.argv.length, 2, 'no arguments required'); const directory = fileURLToPath(new URL('.', import.meta.url)); const root = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-local-package-check-'))); -// The producer and the consumer run detached and clean up after themselves (the producer removes its -// private root on an interrupt), so an interrupt reaches them instead of ending this check first; the -// check root, which holds their temporary directories, goes once they have exited. +// The producer and the consumer run detached, and each stops its own child trees and removes its +// temporary directories on an interrupt, so the interrupt is forwarded to them instead of ending this +// check first. The check root, which holds those directories, goes once they have exited, and no +// later stage starts after an interrupt arrived. const interrupts = forwardInterrupts(); +const proceed = () => + assert.equal(interrupts.received(), undefined, 'PACKED_CLI_LOCAL_CHECK_INTERRUPTED'); try { // The producer runs with an isolated HOME, so the store and the developer's global Git excludes // file are found here, from the caller's own location variables, and handed over explicitly. @@ -23,11 +26,14 @@ try { const artifact = join(root, 'artifact'), reports = join(root, 'reports'); const options = { env: isolatedEnvironment(root), timeoutMs: 600_000 }; + proceed(); await runCommand(process.execPath, [join(directory, 'pack.mjs'), artifact], { ...options, env: producerEnvironment(options.env, { store, excludes }), }); + proceed(); await runCommand(process.execPath, [join(directory, 'consume.mjs'), artifact, reports], options); + proceed(); const record = JSON.parse(readFileSync(join(reports, 'installed-consumer.json'), 'utf8')); assert.equal(record.leg, 'local'); console.log( @@ -44,8 +50,9 @@ try { } catch (error) { if (interrupts.received() === undefined) throw error; } finally { - rmSync(root, { recursive: true, force: true }); - interrupts.dispose(); + // Retries absorb a child killed by the interrupt that was still finishing a write. + rmSync(root, { recursive: true, force: true, maxRetries: 3 }); + await interrupts.dispose(); } const signal = interrupts.received(); if (signal !== undefined) process.exit(128 + (constants.signals[signal] ?? 0)); diff --git a/tools/cli-package/check.test.mjs b/tools/cli-package/check.test.mjs new file mode 100644 index 00000000..0987a445 --- /dev/null +++ b/tools/cli-package/check.test.mjs @@ -0,0 +1,230 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + realpathSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { after, test } from 'node:test'; +import { setTimeout as delay } from 'node:timers/promises'; +import { fileURLToPath } from 'node:url'; + +// The local check runs from a copied tooling tree whose pack.mjs and consume.mjs are stand-ins, so no +// case can build, pack or install the real CLI even if the orchestration regresses. +const tooling = fileURLToPath(new URL('../', import.meta.url)); +const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-local-check-'))); +after(() => rmSync(scratch, { recursive: true, force: true })); +let count = 0; + +const COPIED = Object.freeze([ + 'cli-package/check.mjs', + 'cli-package/candidate.mjs', + 'cli-package/consumer-lifetime.mjs', + 'cli-package/files.mjs', + 'cli-package/finalize.mjs', + 'cli-package/paths.mjs', + 'cli-package/process.mjs', + 'bundle-closure/closure.mjs', + 'bundle-closure/inventory.mjs', +]); + +/** + * A long-lived process that records its pid, then tries to write a file into `directory` every 5 ms; + * it outlives the directory's removal unless it is killed. + */ +function writer(directory, pidFile) { + return ( + `const { writeFileSync } = require('node:fs');` + + `const { join } = require('node:path');` + + `writeFileSync(${JSON.stringify(pidFile)}, String(process.pid));` + + `let n = 0; setInterval(() => { try { writeFileSync(join(${JSON.stringify(directory)}, 'f' + n++), 'x'); } catch {} }, 5);` + ); +} + +/** The consumer stand-in: the real consumer lifetime around a `pnpm add`-like writer in its home. */ +const CONSUMER = (markers) => ` +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { completeConsumer } from './consumer-lifetime.mjs'; +import { isolatedEnvironment, runCommand } from './process.mjs'; +writeFileSync(${JSON.stringify(join(markers, 'consume-started'))}, String(process.pid)); +const home = mkdtempSync(join(tmpdir(), 'relavium-package-consumer-')); +const writer = (${writer.toString()})(home, ${JSON.stringify(join(markers, 'writer.pid'))}); +await completeConsumer({ + exercise: () => + runCommand(process.execPath, ['-e', writer], { + env: isolatedEnvironment(home), + timeoutMs: 60_000, + }), + abandon: () => {}, + cleanup: () => rmSync(home, { recursive: true, force: true, maxRetries: 3 }), + cleanupFailed: () => {}, + publish: () => {}, +}); +`; + +/** + * Start the copied check with stand-ins for its producer and consumer. The store and the excludes + * file are named explicitly, so the check runs neither pnpm nor Git. + */ +function check({ pack }) { + count += 1; + const root = join(scratch, `repository-${count}`); + for (const path of COPIED) { + mkdirSync(dirname(join(root, 'tools', path)), { recursive: true }); + copyFileSync(join(tooling, path), join(root, 'tools', path)); + } + symlinkSync( + fileURLToPath(new URL('../../node_modules', import.meta.url)), + join(root, 'node_modules'), + process.platform === 'win32' ? 'junction' : 'dir', + ); + const markers = join(root, 'markers'); + const temporary = join(root, 'tmp'); + const store = join(root, 'store'); + for (const directory of [markers, temporary, store]) mkdirSync(directory); + const excludes = join(root, 'ignore'); + writeFileSync(excludes, ''); + writeFileSync(join(root, 'tools', 'cli-package', 'pack.mjs'), pack(markers)); + writeFileSync(join(root, 'tools', 'cli-package', 'consume.mjs'), CONSUMER(markers)); + const child = spawn(process.execPath, [join(root, 'tools', 'cli-package', 'check.mjs')], { + env: { + PATH: process.env.PATH, + TMPDIR: temporary, + RELAVIUM_PACKAGE_STORE_DIR: store, + RELAVIUM_GIT_EXCLUDES_FILE: excludes, + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stdout = ''; + child.stdout.on('data', (chunk) => (stdout += chunk)); + const exited = new Promise((resolve) => + child.once('exit', (code, signal) => resolve({ code, signal, stdout })), + ); + return { child, exited, markers, temporary }; +} + +async function waitForFile(file) { + for (let waited = 0; !existsSync(file) || readFileSync(file, 'utf8') === ''; waited += 25) { + assert.ok(waited < 20_000, `${file} never appeared`); + await delay(25); + } + return readFileSync(file, 'utf8'); +} + +function alive(pid) { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +/** + * Stop the check and every stand-in a regressed check could leave behind: the stand-ins run in their + * own process groups, so they would outlive a check that died without forwarding its interrupt. + */ +function stopAll(run) { + if (run.child.exitCode === null && run.child.signalCode === null) run.child.kill('SIGKILL'); + for (const name of ['pack.pid', 'consume-started', 'writer.pid']) { + const file = join(run.markers, name); + const pid = existsSync(file) ? Number(readFileSync(file, 'utf8')) : NaN; + if (Number.isSafeInteger(pid) && pid > 0 && alive(pid)) process.kill(pid, 'SIGKILL'); + } +} + +const checkRoots = (temporary) => + readdirSync(temporary).filter((name) => name.startsWith('relavium-local-package-check-')); + +test( + 'an interrupt during the consumer stops its children, removes the check root and exits', + { skip: process.platform === 'win32' }, + async () => { + const run = check({ + pack: () => `import { mkdirSync } from 'node:fs';\nmkdirSync(process.argv[2]);\n`, + }); + let pid; + try { + pid = Number(await waitForFile(join(run.markers, 'writer.pid'))); + assert.equal(checkRoots(run.temporary).length, 1); + run.child.kill('SIGINT'); + const outcome = await run.exited; + assert.equal(outcome.signal, null); + assert.equal(outcome.code, 130); + assert.equal(outcome.stdout, ''); + // The consumer received the forwarded interrupt and stopped its writer before its home went. + assert.equal(alive(pid), false); + assert.deepEqual(checkRoots(run.temporary), []); + } finally { + stopAll(run); + } + }, +); + +test( + 'an interrupt during the producer reaches it, and the consumer never starts', + { skip: process.platform === 'win32' }, + async () => { + const run = check({ + pack: (markers) => ` +import { writeFileSync } from 'node:fs'; +process.on('SIGINT', () => { + writeFileSync(${JSON.stringify(join(markers, 'pack-interrupted'))}, 'SIGINT'); + process.exit(130); +}); +writeFileSync(${JSON.stringify(join(markers, 'pack.pid'))}, String(process.pid)); +setInterval(() => {}, 1000); +`, + }); + try { + await waitForFile(join(run.markers, 'pack.pid')); + run.child.kill('SIGINT'); + const outcome = await run.exited; + assert.equal(outcome.signal, null); + assert.equal(outcome.code, 130); + assert.equal(readFileSync(join(run.markers, 'pack-interrupted'), 'utf8'), 'SIGINT'); + assert.equal(existsSync(join(run.markers, 'consume-started')), false); + assert.deepEqual(checkRoots(run.temporary), []); + } finally { + stopAll(run); + } + }, +); + +test( + 'an interrupt the producer outlives still stops the check before the consumer', + { skip: process.platform === 'win32' }, + async () => { + // The producer signals the check, ignores the interrupt forwarded back to it and finishes. + const run = check({ + pack: (markers) => ` +import { mkdirSync, writeFileSync } from 'node:fs'; +process.on('SIGINT', () => writeFileSync(${JSON.stringify(join(markers, 'pack-forwarded'))}, 'SIGINT')); +mkdirSync(process.argv[2]); +process.kill(process.ppid, 'SIGINT'); +setTimeout(() => process.exit(0), 500); +`, + }); + try { + const outcome = await run.exited; + assert.equal(outcome.signal, null); + assert.equal(outcome.code, 130); + assert.equal(readFileSync(join(run.markers, 'pack-forwarded'), 'utf8'), 'SIGINT'); + assert.equal(existsSync(join(run.markers, 'consume-started')), false); + assert.deepEqual(checkRoots(run.temporary), []); + } finally { + stopAll(run); + } + }, +); diff --git a/tools/cli-package/consume.mjs b/tools/cli-package/consume.mjs index 581e0e2c..bd5dd84c 100644 --- a/tools/cli-package/consume.mjs +++ b/tools/cli-package/consume.mjs @@ -15,6 +15,7 @@ import { tmpdir } from 'node:os'; import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; import { fileURLToPath } from 'node:url'; import { runBootstrap } from '../cli-bootstrap/check.mjs'; +import { abandonFixtures } from '../cli-bootstrap/child.mjs'; import { inspectPackageArchive } from './archive.mjs'; import { originFromEnvironment, @@ -22,7 +23,13 @@ import { validateConsumerRecord, } from './binding.mjs'; import { installedInventory, oneTarball, readOrdinaryFile, sha256 } from './files.mjs'; -import { exerciseInstalledCommand, isolatedEnvironment, runCommand, runPnpm } from './process.mjs'; +import { + exerciseInstalledCommand, + isolatedEnvironment, + isolatedGitEnvironment, + runCommand, + runPnpm, +} from './process.mjs'; import { runInstalledSmoke } from './smoke/index.mjs'; import { completeConsumer } from './consumer-lifetime.mjs'; @@ -40,7 +47,7 @@ const source = ( await runCommand('git', ['--no-replace-objects', 'rev-parse', 'HEAD'], { cwd: repository, - env: isolatedEnvironment(repository), + env: isolatedGitEnvironment(), }) ).stdout.trim(); const nodeMajor = Number(readFileSync(join(repository, '.nvmrc'), 'utf8').trim()); @@ -176,7 +183,10 @@ await completeConsumer({ validateConsumerRecord(record, receipt, { ...expected, allowLocal: true }); return record; }, - cleanup: () => rmSync(home, { recursive: true, force: true }), + // An interrupt stops the bootstrap's own children and fixtures too; retries absorb a killed child + // that was still finishing a write. + abandon: abandonFixtures, + cleanup: () => rmSync(home, { recursive: true, force: true, maxRetries: 3 }), cleanupFailed: () => writeSync(2, 'PACKED_CLI_SECONDARY_INSTALLATION_CLEANUP_FAILED\n'), publish(record) { writeFileSync( diff --git a/tools/cli-package/consumer-lifetime.mjs b/tools/cli-package/consumer-lifetime.mjs index 1ba77b7b..8f9c9a43 100644 --- a/tools/cli-package/consumer-lifetime.mjs +++ b/tools/cli-package/consumer-lifetime.mjs @@ -1,24 +1,50 @@ -/** A success publication includes actual worker settlement and owned installation removal. */ -export async function completeConsumer({ exercise, cleanup, cleanupFailed, publish }) { - let result; - let primary; - let primaryFailure = false; - try { - result = await exercise(); - } catch (error) { - primary = error; - primaryFailure = true; - } - try { - await cleanup(); - } catch (error) { - if (!primaryFailure) throw error; +import assert from 'node:assert/strict'; +import { exitOnInterrupt } from './process.mjs'; + +/** + * A success publication includes actual worker settlement and owned installation removal. An + * interrupt (SIGINT, SIGTERM, SIGHUP) is the primary outcome: it stops every `runCommand` child tree, + * runs `abandon` for the children and directories `runCommand` does not own, removes the installation + * and exits with the signal's code, reporting a failed removal only through `cleanupFailed`. `abandon` + * and `cleanup` are therefore synchronous. + * @template T + * @param {{exercise: () => Promise, cleanup: () => void, cleanupFailed: () => void, + * publish: (result: T) => unknown, abandon: () => void}} options + */ +export async function completeConsumer({ exercise, cleanup, cleanupFailed, publish, abandon }) { + assert.equal(typeof abandon, 'function', 'PACKED_CLI_CONSUMER_ABANDON'); + const secondary = () => { try { cleanupFailed(); } catch { - // A failed secondary diagnostic cannot replace the original exercise failure. + // A failed secondary diagnostic cannot replace the primary outcome. } - } - if (primaryFailure) throw primary; - return publish(result); + }; + const interrupted = () => { + try { + abandon(); + cleanup(); + } catch { + secondary(); + } + }; + return exitOnInterrupt(interrupted, async () => { + let result; + let primary; + let primaryFailure = false; + try { + result = await exercise(); + } catch (error) { + primary = error; + primaryFailure = true; + } + try { + await cleanup(); + } catch (error) { + if (!primaryFailure) throw error; + secondary(); + } + if (primaryFailure) throw primary; + return publish(result); + }); } diff --git a/tools/cli-package/consumer-lifetime.test.mjs b/tools/cli-package/consumer-lifetime.test.mjs index df66448c..d4e8306e 100644 --- a/tools/cli-package/consumer-lifetime.test.mjs +++ b/tools/cli-package/consumer-lifetime.test.mjs @@ -6,13 +6,16 @@ import { mkdirSync, mkdtempSync, readFileSync, + realpathSync, rmSync, writeFileSync, } from 'node:fs'; +import { spawn } from 'node:child_process'; import { createRequire } from 'node:module'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { test } from 'node:test'; +import { setTimeout as delay } from 'node:timers/promises'; import { pathToFileURL } from 'node:url'; import { completeConsumer } from './consumer-lifetime.mjs'; import { REQUIRED_SMOKE_CASES } from './smoke/cases.mjs'; @@ -44,6 +47,7 @@ async function fixture( join(smoke, 'worker.mjs'), ` import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; import { createRequire } from 'node:module'; import { closeSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; @@ -107,6 +111,7 @@ test('real addon worker closes before installation removal and success publicati // On Windows an in-process or still-running addon owner keeps this copy locked. rmSync(own.installation, { recursive: true, force: true }); }, + abandon: () => {}, cleanupFailed: () => assert.fail('unexpected secondary cleanup failure'), publish: (report) => { assert.equal(existsSync(own.installation), false); @@ -141,6 +146,7 @@ for (const diagnosticSinkFailure of [false, true]) assert.equal(readFileSync(join(own.support, 'worker-exit'), 'utf8'), 'observed'); rmSync(own.installation, { recursive: true, force: true }); }, + abandon: () => {}, cleanupFailed: () => assert.fail('unexpected secondary cleanup failure'), publish: () => { published = true; @@ -186,6 +192,7 @@ for (const [name, options, expected] of [ completeConsumer({ exercise: own.run, cleanup: () => rmSync(own.installation, { recursive: true, force: true }), + abandon: () => {}, cleanupFailed: () => assert.fail('unexpected secondary cleanup failure'), publish: () => { published = true; @@ -207,6 +214,7 @@ test('failed required cleanup prevents publication of a successful exercise', as cleanup: () => { throw cleanupFailure; }, + abandon: () => {}, cleanupFailed: () => assert.fail('cleanup is primary here'), publish: () => { published = true; @@ -231,6 +239,7 @@ for (const primary of [new Error('synthetic worker refusal'), undefined, null]) cleaned = true; throw new Error('synthetic secondary cleanup refusal'); }, + abandon: () => {}, cleanupFailed: () => { diagnosed = true; throw new Error('synthetic failed diagnostic sink'); @@ -245,3 +254,116 @@ for (const primary of [new Error('synthetic worker refusal'), undefined, null]) assert.equal(diagnosed, true); assert.equal(published, false); }); + +test('a consumer without an interrupt cleanup for the children runCommand does not own refuses', async () => { + let exercised = false; + await assert.rejects( + completeConsumer({ + exercise: async () => { + exercised = true; + }, + cleanup: () => {}, + cleanupFailed: () => {}, + publish: () => {}, + }), + /PACKED_CLI_CONSUMER_ABANDON/u, + ); + assert.equal(exercised, false); +}); + +function alive(pid) { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +/** + * A consumer process whose exercise is a long-lived `runCommand` child that keeps writing into the + * owned installation, as `pnpm add` does, and outlives the installation's removal unless it is killed; + * the test interrupts the consumer once the child is running. + */ +async function interruptedConsumer(t, { cleanupFails = false } = {}) { + const root = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-consumer-interrupt-'))); + t.after(() => rmSync(root, { recursive: true, force: true })); + const installation = join(root, 'installation'); + mkdirSync(installation); + const log = join(root, 'events'); + const pidFile = join(root, 'child.pid'); + const url = (name) => JSON.stringify(new URL(name, import.meta.url).href); + const writer = + `const { writeFileSync } = require('node:fs');` + + `writeFileSync(${JSON.stringify(pidFile)}, String(process.pid));` + + `let n = 0; setInterval(() => { try { writeFileSync(require('node:path').join(${JSON.stringify(installation)}, 'f' + n++), 'x'); } catch {} }, 5);`; + const script = ` +import { appendFileSync, rmSync } from 'node:fs'; +import { completeConsumer } from ${url('./consumer-lifetime.mjs')}; +import { isolatedEnvironment, runCommand } from ${url('./process.mjs')}; +const record = (event) => appendFileSync(${JSON.stringify(log)}, event + '\\n'); +await completeConsumer({ + exercise: () => runCommand(process.execPath, ['-e', ${JSON.stringify(writer)}], { + env: isolatedEnvironment(${JSON.stringify(root)}), + timeoutMs: 60_000, + }), + abandon: () => record('abandon'), + cleanup: () => { + record('cleanup'); + if (${cleanupFails}) throw new Error('synthetic cleanup refusal'); + rmSync(${JSON.stringify(installation)}, { recursive: true, force: true, maxRetries: 3 }); + }, + cleanupFailed: () => record('cleanupFailed'), + publish: () => record('publish'), +}); +`; + const consumer = spawn(process.execPath, ['--input-type=module', '-e', script], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + let child; + try { + for ( + let waited = 0; + !existsSync(pidFile) || readFileSync(pidFile, 'utf8') === ''; + waited += 25 + ) { + assert.ok(waited < 20_000, 'the exercise never started'); + await delay(25); + } + child = Number(readFileSync(pidFile, 'utf8')); + const exited = new Promise((resolve) => + consumer.once('exit', (code, signal) => resolve([code, signal])), + ); + consumer.kill('SIGINT'); + const [code, signal] = await exited; + return { code, signal, child, installation, events: readFileSync(log, 'utf8') }; + } finally { + if (consumer.exitCode === null && consumer.signalCode === null) consumer.kill('SIGKILL'); + if (child !== undefined && alive(child)) process.kill(child, 'SIGKILL'); + } +} + +test( + 'an interrupted consumer stops its children, abandons the rest, removes its installation and exits', + { skip: process.platform === 'win32' }, + async (t) => { + const outcome = await interruptedConsumer(t); + assert.equal(outcome.signal, null); + assert.equal(outcome.code, 130); + // The writer was stopped before the removal, which therefore leaves nothing behind. + assert.equal(alive(outcome.child), false); + assert.equal(existsSync(outcome.installation), false); + assert.equal(outcome.events, 'abandon\ncleanup\n'); + }, +); + +test( + 'a failed removal after an interrupt is only the secondary diagnostic', + { skip: process.platform === 'win32' }, + async (t) => { + const outcome = await interruptedConsumer(t, { cleanupFails: true }); + assert.equal(outcome.signal, null); + assert.equal(outcome.code, 130); + assert.equal(outcome.events, 'abandon\ncleanup\ncleanupFailed\n'); + }, +); diff --git a/tools/cli-package/finalize.mjs b/tools/cli-package/finalize.mjs index 42348268..9758085e 100644 --- a/tools/cli-package/finalize.mjs +++ b/tools/cli-package/finalize.mjs @@ -159,7 +159,9 @@ function originalSource(repository, mapDirectory, source, allowedRoots, subject) /** * Validate one consumed upstream (tsc) map. The JS must end with its exact trailer and the map must - * name only this package's original TypeScript, without embedded sources or a source root. + * name only this package's original TypeScript, without embedded sources or a source root. Its + * mappings may be empty only for a module whose types all erase, which tsc emits as exactly + * `export {};`. * @param {string} repository * @param {string} packageName * @param {string} jsPath repository-relative POSIX path of the consumed compiled JS @@ -202,6 +204,9 @@ function upstreamSources(repository, packageName, jsPath) { ) { refuse('UPSTREAM_MAP_SHAPE', mapPath); } + if (map.mappings === '' && js !== `export {};\n${mapTrailer(name)}`) { + refuse('UPSTREAM_MAP_UNMAPPED', mapPath); + } const directory = posix.dirname(jsPath); return map.sources.map((source) => originalSource(repository, directory, source, [`packages/${packageName}/src`], mapPath), diff --git a/tools/cli-package/finalize.test.mjs b/tools/cli-package/finalize.test.mjs index 74f6ae02..80e76f17 100644 --- a/tools/cli-package/finalize.test.mjs +++ b/tools/cli-package/finalize.test.mjs @@ -255,6 +255,8 @@ for (const [label, change, code] of [ ['another file', (map) => (map.file = 'other.js'), 'UPSTREAM_MAP_SHAPE'], ['no sources', (map) => (map.sources = []), 'UPSTREAM_MAP_SHAPE'], ['malformed mappings', (map) => (map.mappings = 'AAAA!'), 'UPSTREAM_MAP_SHAPE'], + // Only a module whose types all erase (`export {};`, as types.js is) may map nothing. + ['no mappings for executable code', (map) => (map.mappings = ''), 'UPSTREAM_MAP_UNMAPPED'], [ 'a URL source', (map) => (map.sources = ['https://example.invalid/index.ts']), diff --git a/tools/cli-package/prepack.test.mjs b/tools/cli-package/prepack.test.mjs index 95c159ed..ddc6e84e 100644 --- a/tools/cli-package/prepack.test.mjs +++ b/tools/cli-package/prepack.test.mjs @@ -24,7 +24,7 @@ const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-prepack-orches after(() => rmSync(scratch, { recursive: true, force: true })); let count = 0; -function fixture({ buildExit = 0 } = {}) { +function fixture({ buildExit = 0, tsup: installed = true } = {}) { count += 1; const root = join(scratch, `repository-${count}`); for (const path of [ @@ -46,13 +46,15 @@ function fixture({ buildExit = 0 } = {}) { ); const cli = join(root, 'apps', 'cli'); const tsup = join(cli, 'node_modules', 'tsup'); - mkdirSync(join(tsup, 'dist'), { recursive: true }); + const marker = join(root, 'build-ran'); + mkdirSync(cli, { recursive: true }); writeFileSync(join(cli, 'package.json'), JSON.stringify({ name: 'relavium', dependencies: {} })); + if (!installed) return { root, cli, marker }; + mkdirSync(join(tsup, 'dist'), { recursive: true }); writeFileSync( join(tsup, 'package.json'), JSON.stringify({ name: 'tsup', type: 'module', bin: { tsup: 'dist/cli.js' } }), ); - const marker = join(root, 'build-ran'); writeFileSync( join(tsup, 'dist', 'cli.js'), `import { appendFileSync } from 'node:fs';\n` + @@ -132,3 +134,15 @@ test('one build runs, then the ordinary closure gate or the candidate finalizer ); assert.doesNotMatch(mapped.stdout, /CLI candidate finalized/u); }); + +test('an unexpected error prints only its code, never its message or a host path', () => { + // Without an installed tsup the resolution fails with an error whose message names absolute + // fixture paths; only its code may reach the pack log. + const candidate = fixture({ tsup: false }); + const staging = privateStaging(); + const result = prepack(candidate, { staging }); + assert.equal(result.status, 1); + assert.equal(result.stdout, ''); + assert.equal(result.stderr, '✗ CLI candidate finalization failed: MODULE_NOT_FOUND\n'); + assert.equal(existsSync(candidate.marker), false); +}); diff --git a/tools/cli-package/process.mjs b/tools/cli-package/process.mjs index 232b20d7..efb5771b 100644 --- a/tools/cli-package/process.mjs +++ b/tools/cli-package/process.mjs @@ -3,8 +3,9 @@ import assert from 'node:assert/strict'; import { spawn, spawnSync } from 'node:child_process'; import { Buffer } from 'node:buffer'; import { existsSync, lstatSync, readFileSync, realpathSync, statSync } from 'node:fs'; +import { constants, devNull } from 'node:os'; import { delimiter, extname, isAbsolute, join } from 'node:path'; -import { clearTimeout, setTimeout } from 'node:timers'; +import { clearTimeout, setImmediate, setTimeout } from 'node:timers'; export function isolatedEnvironment(home) { const env = { @@ -25,6 +26,16 @@ export function isolatedEnvironment(home) { return env; } +/** + * Git with no global or system configuration, for reading the original checkout: its own `.git/config` + * and ignore rules still apply and a caller passes the developer's global excludes explicitly, but no + * `.gitconfig`, `git/config`, `git/ignore` or `git/attributes` in a home, configuration or system + * location is read — not even one inside the checkout itself. + */ +export function isolatedGitEnvironment() { + return { ...isolatedEnvironment(devNull), GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: devNull }; +} + /** Resolve reviewed pnpm JS shims or the action's standalone executable, without cmd.exe. */ export function resolvePnpm(path = process.env.PATH ?? '', platform = process.platform) { for (const directory of path.split(delimiter).filter(Boolean)) { @@ -93,16 +104,54 @@ export function stopLiveCommands() { liveChildren.clear(); } -/** The interrupts a tool forwards to the children that own their cleanup. */ -const FORWARDED_INTERRUPTS = Object.freeze(['SIGINT', 'SIGTERM', 'SIGHUP']); +/** The interrupts after which a tool still stops its children and cleans up before it exits. */ +const INTERRUPTS = Object.freeze(['SIGINT', 'SIGTERM', 'SIGHUP']); + +/** + * Let the event loop poll for I/O once more. Node hands a caught signal to its listener only when the + * loop next polls, and work that ran from an I/O callback (a child's exit, a file read) can end its + * first turn before that poll: a listener removed then drops the signal. After two turns the loop has + * polled. A signal caught in the instant between that poll and the listener's removal is still lost; + * Node offers no way to drain it. + */ +async function deliverCaughtSignals() { + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); +} + +/** + * Run `body` while SIGINT, SIGTERM and SIGHUP stop every child tree `runCommand` owns, run the + * synchronous `interrupted` and end the process with 128 + the signal's number. The handlers stay + * until the loop has polled once after `body` settled (`deliverCaughtSignals`), so an interrupt caught + * during the body's synchronous tail or its own final cleanup is honoured, never dropped with its + * listener. A process killed outright runs nothing. + * @template T + * @param {() => void} interrupted + * @param {() => Promise} body + * @returns {Promise} + */ +export async function exitOnInterrupt(interrupted, body) { + const handler = (signal) => { + stopLiveCommands(); + interrupted(); + process.exit(128 + (constants.signals[signal] ?? 0)); + }; + for (const signal of INTERRUPTS) process.on(signal, handler); + try { + return await body(); + } finally { + await deliverCaughtSignals(); + for (const signal of INTERRUPTS) process.off(signal, handler); + } +} /** - * While a tool waits on children that clean up after themselves (the producer), forward SIGINT, - * SIGTERM and SIGHUP to every child tree `runCommand` owns instead of dying first: a detached child - * shares no terminal process group, so it would never receive the interrupt. On Windows the children - * share the console and receive Ctrl-C themselves. The caller runs its own cleanup, calls `dispose` + * While a tool waits on children that clean up after themselves (the producer, the consumer), + * forward SIGINT, SIGTERM and SIGHUP to every child tree `runCommand` owns instead of dying first: a + * detached child shares no terminal process group, so it would never receive the interrupt. On + * Windows the children share the console and receive Ctrl-C themselves. The caller runs its own cleanup, awaits `dispose` * and exits with 128 + the number of the signal `received` reports. - * @returns {{received: () => string | undefined, dispose: () => void}} + * @returns {{received: () => string | undefined, dispose: () => Promise}} */ export function forwardInterrupts() { let received; @@ -117,11 +166,13 @@ export function forwardInterrupts() { } } }; - for (const signal of FORWARDED_INTERRUPTS) process.on(signal, forward); + for (const signal of INTERRUPTS) process.on(signal, forward); return { received: () => received, - dispose: () => { - for (const signal of FORWARDED_INTERRUPTS) process.off(signal, forward); + // An interrupt caught during the caller's cleanup is still recorded (`deliverCaughtSignals`). + dispose: async () => { + await deliverCaughtSignals(); + for (const signal of INTERRUPTS) process.off(signal, forward); }, }; } diff --git a/tools/cli-package/process.test.mjs b/tools/cli-package/process.test.mjs index cf9922a6..496100a6 100644 --- a/tools/cli-package/process.test.mjs +++ b/tools/cli-package/process.test.mjs @@ -11,16 +11,28 @@ import { rmSync, realpathSync, } from 'node:fs'; -import { tmpdir } from 'node:os'; +import { devNull, tmpdir } from 'node:os'; import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; import { exerciseInstalledCommand, isolatedEnvironment, + isolatedGitEnvironment, resolvePnpm, runCommand, runPnpm, } from './process.mjs'; +test('Git reading the original checkout gets no home, global or system configuration', () => { + const env = isolatedGitEnvironment(); + assert.deepEqual(env, { + ...isolatedEnvironment(devNull), + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: devNull, + }); + for (const key of ['HOME', 'USERPROFILE', 'XDG_CONFIG_HOME']) assert.equal(env[key], devNull); +}); + test('installer receives an explicit credential/config/preload-free environment', () => { const home = join(tmpdir(), 'synthetic-installer-home'); const env = isolatedEnvironment(home); @@ -208,7 +220,7 @@ try { } catch (error) { if (interrupts.received() === undefined) throw error; } finally { - interrupts.dispose(); + await interrupts.dispose(); } const signal = interrupts.received(); if (signal !== undefined) process.exit(128 + constants.signals[signal]); @@ -236,3 +248,39 @@ if (signal !== undefined) process.exit(128 + constants.signals[signal]); } }, ); + +test( + 'an interrupt caught during the cleanup after the last child is still recorded', + { skip: process.platform === 'win32' }, + async () => { + // The last await completes from an I/O callback, so the cleanup runs inside the loop's poll + // phase: a listener removed after one more turn would drop the caught signal. + const harness = ` +import { readFile } from 'node:fs/promises'; +import { constants } from 'node:os'; +import { forwardInterrupts } from ${JSON.stringify(new URL('./process.mjs', import.meta.url).href)}; +const interrupts = forwardInterrupts(); +try { + await readFile(${JSON.stringify(fileURLToPath(import.meta.url))}); +} finally { + process.kill(process.pid, 'SIGINT'); + for (const until = Date.now() + 50; Date.now() < until; ); + await interrupts.dispose(); +} +const signal = interrupts.received(); +if (signal !== undefined) process.exit(128 + constants.signals[signal]); +console.log('not interrupted'); +`; + const tool = spawn(process.execPath, ['--input-type=module', '-e', harness], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stdout = ''; + tool.stdout.on('data', (chunk) => (stdout += chunk)); + const [code, signal] = await new Promise((resolve) => + tool.once('exit', (exitCode, signalCode) => resolve([exitCode, signalCode])), + ); + assert.equal(signal, null); + assert.equal(code, 130); + assert.equal(stdout, ''); + }, +); diff --git a/tools/cli-package/produce.mjs b/tools/cli-package/produce.mjs index 3a73b4b1..dad134d6 100644 --- a/tools/cli-package/produce.mjs +++ b/tools/cli-package/produce.mjs @@ -10,24 +10,22 @@ */ import assert from 'node:assert/strict'; import { chmodSync, mkdirSync, mkdtempSync, readdirSync, realpathSync, rmSync } from 'node:fs'; -import { constants, tmpdir } from 'node:os'; +import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { assertNoBuildCache, globalExcludesFile, prepareCandidate } from './candidate.mjs'; import { within } from './paths.mjs'; -import { isolatedEnvironment, runPnpm, stopLiveCommands } from './process.mjs'; +import { exitOnInterrupt, isolatedEnvironment, runPnpm } from './process.mjs'; import { sealPackage } from './seal.mjs'; import { emitFreshPackage } from './pipeline.mjs'; -/** The interrupts after which the producer still removes everything private before it exits. */ -const INTERRUPTS = Object.freeze(['SIGINT', 'SIGTERM', 'SIGHUP']); - /** * Produce one package-test artifact from the original checkout `repository` into the empty * `destination`. The candidate tree and its private staging are siblings in one owner-only temporary * root that shares no path with the artifact directory or the checkout. That root and the build home - * are removed whether production succeeds or fails; an interrupt also stops the build's child trees - * and empties the artifact directory, which was empty at the start, before the process exits with - * the signal's code. + * are removed whether production succeeds or fails. An interrupt (SIGINT, SIGTERM, SIGHUP) stops the + * build's child trees, removes both directories and empties the artifact directory, which was empty + * at the start, before the process exits with the signal's code — also when it arrives after the last + * child exited, during the seal or during the final removal (`exitOnInterrupt`). * @param {{repository: string, destination: string, environment: Record, * temporary?: string, ports?: {prepare?: typeof prepareCandidate, emit?: typeof emitFreshPackage, * seal?: typeof sealPackage, noBuildCache?: typeof assertNoBuildCache, @@ -48,7 +46,8 @@ export async function producePackage({ noBuildCache = assertNoBuildCache, excludesFile = globalExcludesFile, pnpm = runPnpm, - remove = (path) => rmSync(path, { recursive: true, force: true }), + // Retries absorb a child killed by the interrupt that was still finishing a write. + remove = (path) => rmSync(path, { recursive: true, force: true, maxRetries: 3 }), } = ports; mkdirSync(destination, { recursive: true }); assert.deepEqual(readdirSync(destination), [], 'PACKED_CLI_FRESH_DESTINATION_REQUIRED'); @@ -58,61 +57,59 @@ export async function producePackage({ if (root !== undefined) remove(root); remove(home); }; - const interrupt = (signal) => { - stopLiveCommands(); + const interrupted = () => { discard(); for (const name of readdirSync(destination)) { - rmSync(join(destination, name), { recursive: true, force: true }); + rmSync(join(destination, name), { recursive: true, force: true, maxRetries: 3 }); } - process.exit(128 + (constants.signals[signal] ?? 0)); }; - for (const signal of INTERRUPTS) process.on(signal, interrupt); - try { - root = realpathSync.native(mkdtempSync(join(temporary, 'relavium-package-candidate-'))); - chmodSync(root, 0o700); - const candidate = join(root, 'source'); - const staging = join(root, 'staging'); - mkdirSync(candidate, { mode: 0o700 }); - mkdirSync(staging, { mode: 0o700 }); - chmodSync(staging, 0o700); - // Native real paths carry the letter case the filesystem stores, so no alias passes for elsewhere. - const published = realpathSync.native(destination); - const checkout = realpathSync.native(repository); - assert.ok( - !within(root, published) && !within(published, root) && !within(checkout, root), - 'PACKED_CLI_PRIVATE_STAGING_PLACEMENT', - ); - const env = isolatedEnvironment(home); - const version = await pnpm(['--version'], { cwd: repository, env }); - assert.equal(version.stdout.trim(), '9.12.3', 'PACKED_CLI_PNPM_VERSION'); + return exitOnInterrupt(interrupted, async () => { + try { + root = realpathSync.native(mkdtempSync(join(temporary, 'relavium-package-candidate-'))); + chmodSync(root, 0o700); + const candidate = join(root, 'source'); + const staging = join(root, 'staging'); + mkdirSync(candidate, { mode: 0o700 }); + mkdirSync(staging, { mode: 0o700 }); + chmodSync(staging, 0o700); + // Native real paths carry the letter case the filesystem stores, so no alias passes for elsewhere. + const published = realpathSync.native(destination); + const checkout = realpathSync.native(repository); + assert.ok( + !within(root, published) && !within(published, root) && !within(checkout, root), + 'PACKED_CLI_PRIVATE_STAGING_PLACEMENT', + ); + const env = isolatedEnvironment(home); + const version = await pnpm(['--version'], { cwd: repository, env }); + assert.equal(version.stdout.trim(), '9.12.3', 'PACKED_CLI_PNPM_VERSION'); - const excludes = await excludesFile({ repository, environment }); - const identity = await prepare({ repository, candidate, environment, home, env, excludes }); - const receipt = await emit({ - repository: candidate, - destination, - staging, - env, - pnpm, - removeBuildState: (cwd) => { - rmSync(join(cwd, 'dist'), { recursive: true, force: true }); - rmSync(join(cwd, 'tsconfig.build.tsbuildinfo'), { force: true }); - }, - seal: (artifactDirectory) => { - noBuildCache(candidate); - return seal(artifactDirectory, { - repository: candidate, - staging, - source: identity.source, - dirty: identity.dirty, - }); - }, - }); - return { receipt, identity }; - } finally { - // The handlers stay until everything private is gone: a late interrupt waits for the removal - // instead of ending the process in the middle of it. - discard(); - for (const signal of INTERRUPTS) process.off(signal, interrupt); - } + const excludes = await excludesFile({ repository, environment }); + const identity = await prepare({ repository, candidate, environment, home, env, excludes }); + const receipt = await emit({ + repository: candidate, + destination, + staging, + env, + pnpm, + removeBuildState: (cwd) => { + rmSync(join(cwd, 'dist'), { recursive: true, force: true }); + rmSync(join(cwd, 'tsconfig.build.tsbuildinfo'), { force: true }); + }, + seal: (artifactDirectory) => { + noBuildCache(candidate); + return seal(artifactDirectory, { + repository: candidate, + staging, + source: identity.source, + dirty: identity.dirty, + }); + }, + }); + return { receipt, identity }; + } finally { + // The interrupt handler is still installed: an interrupt during this removal is honoured once + // it is done, never in the middle of it. + discard(); + } + }); } diff --git a/tools/cli-package/produce.test.mjs b/tools/cli-package/produce.test.mjs index ad38c3e2..696e4ad4 100644 --- a/tools/cli-package/produce.test.mjs +++ b/tools/cli-package/produce.test.mjs @@ -16,6 +16,7 @@ import { basename, dirname, join } from 'node:path'; import { after, test } from 'node:test'; import { setTimeout as delay } from 'node:timers/promises'; import { assertNoBuildCache } from './candidate.mjs'; +import { isolatedEnvironment } from './process.mjs'; import { producePackage } from './produce.mjs'; const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-produce-'))); @@ -41,8 +42,9 @@ function ports({ plantCache = false, failEmit = false } = {}) { return { seen, ports: { - pnpm: async (args) => { + pnpm: async (args, options) => { assert.deepEqual(args, ['--version']); + seen.pnpm = options; return { stdout: '9.12.3\n', stderr: '' }; }, excludesFile: async ({ repository }) => { @@ -82,20 +84,44 @@ function leftovers(temporary) { const INTERRUPTS = ['SIGINT', 'SIGTERM', 'SIGHUP']; const listeners = () => INTERRUPTS.map((signal) => process.listenerCount(signal)); -test('the producer builds and seals only the owned candidate, never the original checkout', async () => { +/** Credentials, a preload and a registry in the caller's own environment, which no build may see. */ +const PLANTED = Object.freeze({ + NPM_TOKEN: 'planted-npm-token', + NODE_OPTIONS: '--require=/planted/preload.cjs', + npm_config_registry: 'https://registry.planted.invalid/', +}); + +test('the producer builds and seals only the owned candidate, never the original checkout', async (t) => { const repository = fresh('original-checkout'); const destination = join(fresh('artifact-parent'), 'artifact'); const temporary = fresh('tmp'); const { seen, ports: fakes } = ports(); const before = listeners(); + const saved = Object.fromEntries(Object.keys(PLANTED).map((key) => [key, process.env[key]])); + t.after(() => { + for (const [key, value] of Object.entries(saved)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + }); + Object.assign(process.env, PLANTED); const result = await producePackage({ repository, destination, - environment: { HOME: '/home/someone' }, + environment: { HOME: '/home/someone', NPM_TOKEN: 'environment-npm-token' }, temporary, ports: fakes, }); assert.deepEqual(result.identity, { source: SHA, dirty: false, files: 7 }); + // The whole build — pnpm, the candidate's preparation and the emit with its pack — runs in the + // isolated environment of the private build home, never with the caller's own variables. + const home = seen.prepare.home; + assert.equal(dirname(home), temporary); + assert.match(basename(home), /^relavium-package-build-/u); + assert.deepEqual(seen.prepare.env, isolatedEnvironment(home)); + assert.deepEqual(seen.emit.env, seen.prepare.env); + assert.deepEqual(seen.pnpm.env, seen.prepare.env); + for (const key of Object.keys(PLANTED)) assert.equal(key in seen.prepare.env, false); const candidate = seen.prepare.candidate; // The candidate and its staging are siblings in one private root outside the checkout and the artifact. assert.notEqual(candidate, repository); @@ -206,6 +232,20 @@ test('private staging never shares a path with the checkout or the artifact', as /PRIVATE_STAGING_PLACEMENT/u, ); assert.deepEqual(leftovers(inside), []); + // The same holds when the checkout itself is named in another letter case. + const { seen: cased, ports: casedFakes } = ports(); + await assert.rejects( + producePackage({ + repository: variant, + destination: join(fresh('artifact-parent'), 'artifact'), + environment: {}, + temporary: inside, + ports: casedFakes, + }), + /PRIVATE_STAGING_PLACEMENT/u, + ); + assert.equal(cased.prepare, undefined); + assert.deepEqual(leftovers(inside), []); } // So would a temporary directory inside the artifact directory. const artifact = join(fresh('artifact-parent'), 'artifact'); @@ -330,18 +370,23 @@ for (const [signal, code] of [ ); } -test( - 'an interrupt during the final removal waits for it instead of ending the process', - { skip: !posixModes }, - async () => { - const temporary = fresh('tmp'); - mkdirSync(join(temporary, 'checkout')); - const destination = join(fresh('artifact-parent'), 'artifact'); - const url = (name) => JSON.stringify(new URL(name, import.meta.url).href); - // The private root's removal itself is interrupted; the handlers must still be installed then. - const script = ` -import { rmSync } from 'node:fs'; +/** + * A producer whose interrupt arrives after its build's last child exited: during the seal or during + * the private root's removal, both synchronous. The fake emit's last await completes from an I/O + * callback, as pnpm pack's exit does, so the producer's tail runs inside the event loop's poll phase. + */ +function lateInterrupt(temporary, destination, during) { + const url = (name) => JSON.stringify(new URL(name, import.meta.url).href); + const script = ` +import { readFile } from 'node:fs/promises'; +import { rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; import { producePackage } from ${url('./produce.mjs')}; +const interrupt = () => { + process.kill(process.pid, 'SIGINT'); + // The rest of the synchronous work, during which the signal is only caught. + for (const until = Date.now() + 50; Date.now() < until; ); +}; await producePackage({ repository: ${JSON.stringify(join(temporary, 'checkout'))}, destination: ${JSON.stringify(destination)}, @@ -351,22 +396,51 @@ await producePackage({ pnpm: async () => ({ stdout: '9.12.3\\n', stderr: '' }), excludesFile: async () => undefined, prepare: async () => ({ source: '${SHA}', dirty: false, files: 1 }), - emit: async ({ staging }) => ({ staging }), + emit: async ({ staging, destination, seal }) => { + writeFileSync(join(staging, 'index.js.map'), 'private map'); + writeFileSync(join(destination, 'relavium-0.1.1.tgz'), 'tarball'); + await readFile(join(staging, 'index.js.map')); + return seal(destination); + }, + noBuildCache: () => {}, + seal: () => { + if (${JSON.stringify(during)} === 'seal') interrupt(); + return { mode: 'compiled_package_test' }; + }, remove: (path) => { - if (path.includes('relavium-package-candidate-')) process.kill(process.pid, 'SIGINT'); + if (${JSON.stringify(during)} === 'removal' && path.includes('relavium-package-candidate-')) + interrupt(); rmSync(path, { recursive: true, force: true }); }, }, }); +console.log('produced'); `; - const child = spawn(process.execPath, ['--input-type=module', '-e', script], { - stdio: ['ignore', 'pipe', 'pipe'], - }); - const [code, signal] = await new Promise((resolve) => - child.once('exit', (exitCode, signalCode) => resolve([exitCode, signalCode])), - ); - assert.equal(signal, null); - assert.equal(code, 0); - assert.deepEqual(leftovers(temporary), []); - }, -); + return spawn(process.execPath, ['--input-type=module', '-e', script], { + stdio: ['ignore', 'pipe', 'pipe'], + }); +} + +for (const during of ['seal', 'removal']) + test( + `an interrupt during the ${during}, after the last child exited, still ends the producer`, + { skip: !posixModes }, + async () => { + const temporary = fresh('tmp'); + mkdirSync(join(temporary, 'checkout')); + const destination = join(fresh('artifact-parent'), 'artifact'); + const child = lateInterrupt(temporary, destination, during); + let stdout = ''; + child.stdout.on('data', (chunk) => (stdout += chunk)); + const [code, signal] = await new Promise((resolve) => + child.once('exit', (exitCode, signalCode) => resolve([exitCode, signalCode])), + ); + // Honoured once the synchronous work is done: the removal completes, the artifact is emptied + // and the producer exits with the signal's code instead of reporting success. + assert.equal(signal, null); + assert.equal(code, 130); + assert.equal(stdout, ''); + assert.deepEqual(leftovers(temporary), []); + assert.deepEqual(readdirSync(destination), []); + }, + ); From d51a2437dba671f1a69643e88777847f4d0ea873 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Thu, 22 Oct 2026 11:42:22 +0300 Subject: [PATCH 26/97] fix(repo): address the fourth Step 9a.1 review round Round 4 (at aa5c994a) reported ten verified findings, three medium; its integration and critic lenses were clean. - An interrupted consumer killed the installed smoke's worker before its own removal: the worker's directory stayed under the report directory and the native fixture target the installed CLI had started in its own process group ran on as an orphan. The fixture's processes now end once the smoke worker that owns their case is gone, and completeConsumer, given the report directory as a required supportRoot, abandons the bootstrap witness's children and fixtures and the smoke's worker roots before removing the installation. Interrupting the real consumer during the native cases leaves no process and no directory. - The local check's ordinary failures, its summary and its hand-off of the store and excludes variables are tested. - A dirty identity's path to the seal, the producer's removal of engine build state and the bounded children's process group are tested. - "Owner-only" is qualified where POSIX modes are not observable; the candidate copy's umask wording and the checkpoint's scope list are corrected, and the register records round 4. Gates: lint:tools, typecheck:tools, every pnpm run ci step except the three control families (their captured sources are unchanged), with the real producer and consumer in smoke:cli-package, and the acceptance tools under CI=true GITHUB_ACTIONS=true; a mutation run of fifteen mutants over the corrections kills all of them. Refs: ADR-0111, ADR-0051 Co-Authored-By: Claude Opus 5.5 --- docs/reference/cli/packaging.md | 22 ++- .../phase-2.5.5-hardening-and-remediation.md | 10 +- tools/cli-bootstrap/witness.test.mjs | 61 ++++--- tools/cli-package/README.md | 6 +- tools/cli-package/candidate.mjs | 3 +- tools/cli-package/candidate.test.mjs | 29 +++- tools/cli-package/check.test.mjs | 125 +++++++++++++- tools/cli-package/consume.mjs | 7 +- tools/cli-package/consumer-lifetime.mjs | 32 ++-- tools/cli-package/consumer-lifetime.test.mjs | 162 ++++++++++++------ tools/cli-package/produce.mjs | 5 +- tools/cli-package/produce.test.mjs | 27 ++- tools/cli-package/smoke/README.md | 4 +- tools/cli-package/smoke/index.mjs | 18 ++ tools/cli-package/smoke/native-descendant.cjs | 12 +- tools/cli-package/smoke/native-process.mjs | 2 + .../cli-package/smoke/native-process.test.mjs | 77 +++++++++ tools/cli-package/smoke/native-status.cjs | 13 +- tools/cli-package/smoke/worker.mjs | 3 +- 19 files changed, 482 insertions(+), 136 deletions(-) diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 2410f25a..7101a80f 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -258,10 +258,10 @@ describes the implemented mechanism; independent acceptance is recorded in the directory it then runs the ordinary closure guard, as before. The CLI configuration emits external maps with `sourcesContent` disabled whenever `RELAVIUM_CLI_CANDIDATE_STAGING` is defined; prepack first refuses unless it names an - absolute, canonical, empty, owner-only directory outside the candidate (both compared by - the real paths and letter case the filesystem stores), and then runs the finalizer - in place of the ordinary guard. That guard refuses any map a build outside prepack - leaves in `dist`. + absolute, canonical, empty directory outside the candidate (both compared by the real + paths and letter case the filesystem stores), owner-only where POSIX modes are + observable, and then runs the finalizer in place of the ordinary guard. That guard + refuses any map a build outside prepack leaves in `dist`. - **Finalizer.** Before any write it accounts for every emitted file (each runtime JS output paired with exactly one map, plus the metafile, nothing else). It validates every consumed upstream tsc map: exact terminal map link, version 3, matching `file`, empty @@ -291,12 +291,18 @@ producer succeeds, fails or is interrupted (SIGINT, SIGTERM, SIGHUP); an interru the build's child processes and empties the artifact directory, which was empty at the start, before the producer exits with the signal's code — also when it arrives after the build's last child exited, during the seal or the final removal. An interrupted consumer likewise stops its -child processes, including the bootstrap witness's, and removes its installation and fixtures. -The local `pnpm smoke:cli-package` forwards an interrupt to the producer or consumer it started +child processes, including the bootstrap witness's, and removes its installation, the witness's +fixtures and the installed smoke's working directory; the native fixture targets the installed CLI +starts in their own process groups end on their own once the smoke worker that owns their case is +gone. The local `pnpm smoke:cli-package` forwards an interrupt to the producer or consumer it started rather than ending first, starts no later stage once one arrived, and exits with the signal's code after removing its root. A producer killed outright, a crash or a lost host can leave the -owner-only root in the temporary directory, and no secure erasure is claimed. The private map manifest, actual packed CLI and engine -symbolication, `check:map-controls` and the `package_test` receipt flip are Step 9a.2; +owner-only root in the temporary directory, and no secure erasure is claimed. "Owner-only" in this +section means mode 0700 for directories and 0600 for staged files, set — and for the staging +directory verified together with this process's uid — where POSIX modes are observable. On Windows +no mode evidence is claimed: the candidate root and staging inherit the temporary directory's ACL, +which is not verified, as the graph checks above claim no unobservable POSIX mode bits. The +private map manifest, actual packed CLI and engine symbolication, `check:map-controls` and the `package_test` receipt flip are Step 9a.2; recipient custody, the release state controller and the workflow graph are Steps 9b–9d. Until those and Wave 4b's production recipient exist, the release workflow's publish job refuses every artifact (the [Interim release guard ruling](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) of 2026-10-19). diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index de178d6d..fb25d0db 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -9539,8 +9539,11 @@ corrections. It changes point `pack.mjs`; the prepack orchestrator `prepack.mjs`; the finalizer `finalize.mjs`; the seal; the archive and smoke fold-ins; the interim publication guard `release-guard.mjs` with its rule `release-guard-core.mjs`), `tools/bundle-closure` (map-aware closure and a shared emitted inventory), -`apps/cli` (the prepack script and the candidate-mode tsup selection) and `.github/workflows/release.yml` -(the guard before npm publication), with the +`apps/cli` (the prepack script and the candidate-mode tsup selection), `.github/workflows/release.yml` +(the guard before npm publication) and, through the review rounds, `tools/cli-bootstrap` (stopping the +bootstrap witness's bounded children and removing its live fixtures for an interrupted consumer), the +installed smoke's native fixtures and worker root, the `ci.yml` pack-step comment, the acceptance-tools +list in `package.json` and `tools/ci/require-packed-cli.test.mjs`, with the [packaging contract](../../reference/cli/packaging.md#candidate-build-and-finalizer-implementation-boundary--step-9a1), the release runbook, an ADR-0051 amendment and the tooling README. It adds no runtime dependency, and the receipt stays `compiled_package_test`; the contract section is the canonical description of the @@ -9554,7 +9557,7 @@ entry point, so a symlinked or otherwise non-canonical path would have let a tag global Git excludes, so a globally ignored private file entered the candidate; and smaller gaps in tests, modes and wording. All are corrected: the guard's entry point exits 1 before it evaluates its rule, the producer is a tested module, the candidate reads the developer's global excludes file and nothing else -of the global configuration, the candidate copy's modes are exact under any umask, and each gap has a +of the global configuration, the candidate copy's modes are exact under any umask that lets the owner write, and each gap has a killing test. A mutation run of twenty mutants over the new code kills all of them. The full local CI then found one more defect in the change itself: `tools/bundle-closure/closure.test.mjs` built its staged fixture with a top-level `await` after most of its tests were registered, so a slow build let the @@ -9581,6 +9584,7 @@ adversarially verified; the detailed reports stay local and Git-ignored. | 1 | `a72c7ecd` | NONCLEAN (plan lens clean) | 9 (2 medium, 6 low, 1 nit) | an interrupted producer (SIGINT, SIGTERM, SIGHUP) left its private maps and candidate tree in the temporary directory while its detached build ran on, so the producer now stops the child trees `runCommand` tracks and removes the private root, the build home and the artifact directory's contents before exiting with the signal's code (a producer killed outright is a named limit); private staging must be canonical and outside the candidate by real path; the global-excludes wiring, the producer environment, the placement check's artifact half, the candidate's bounds and three finalizer refusals gained killing tests; the status lines, the CI pack-step comment and the post-closure CI record were corrected | | 2 | `8ddb7ec2` | NONCLEAN (plan and integration lenses clean) | 12 (4 medium, 7 low, 1 nit; 9 distinct) | the producer removed its interrupt handlers before the final removal, so an interrupt during it ended the process with the private root half removed: the handlers now stay until everything private is gone, and a deterministic test interrupts that removal; SIGHUP, the handlers' removal and a whole build tree (a grandchild in the build's process group) are tested; path containment is decided by segments in one shared helper (`..private` is inside); private staging and the producer's placement compare the real paths and letter case the filesystem stores (`realpathSync.native`); the local `pnpm smoke:cli-package` forwards an interrupt to the producer and the consumer it started (`forwardInterrupts`); the finalizer creates its maps directory with an exact mode and its staged and public modes are tested under a umask that removes the owner's write bit; further finalizer and staged-closure refusals gained killing tests; the ADR-0051 note links ADR-0111 | | 3 | `184562bd` | NONCLEAN (security lens clean) | 11 (2 medium, 8 low, 1 nit) | round 2 still dropped an interrupt that arrived after the build's last child exited, during the seal or the final removal, because Node delivers a caught signal only when the event loop next polls and the handlers were removed first; round 2's test asserted exit 0 and so pinned the dropped interrupt. Every tool now keeps its handlers until the loop has polled once more, which takes two turns when the work ran from an I/O callback (one turn, the suggested fix, was measured to drop it), and the producer then empties the artifact directory and exits with the signal's code, tested from an I/O-callback tail. An interrupted consumer stops its own and the bootstrap witness's children and removes its installation and fixtures, and the local `pnpm smoke:cli-package`, now tested end to end, starts no later stage once an interrupt arrived. The build's isolated environment, the producer's native real path of the checkout and the prepack catch-all gained killing tests. Git reading the checkout uses no home, global or system configuration, a skip-worktree or assume-unchanged file makes the identity dirty, and an empty upstream mapping is accepted only for `export {};` | +| 4 | `aa5c994a` | NONCLEAN (integration and critic lenses clean) | 10 (3 medium, 5 low, 2 nits) | an interrupted consumer killed the installed smoke's worker before its own removal, so the worker's directory (42 MB) stayed under the report directory and the native fixture target the installed CLI had started in its own process group ran on as an orphan: the fixture's processes now end once the smoke worker that owns their case is gone, and the consumer lifetime, given the report directory as a required `supportRoot`, abandons the bootstrap witness's children and fixtures and the smoke's worker roots itself before removing the installation. The local check's ordinary failures, its successful summary and its hand-off of the store and excludes variables, a dirty identity's path to the seal, the producer's removal of engine build state and the bounded children's process group gained killing tests. "Owner-only" is qualified where POSIX modes are not observable (Windows); the candidate copy's umask wording and the checkpoint's scope list were corrected | **Remaining gates.** The increment is not independently accepted. Two NEW consecutive clean complete cumulative rounds after the last material fix and a separate security sitting remain (the Acceptance diff --git a/tools/cli-bootstrap/witness.test.mjs b/tools/cli-bootstrap/witness.test.mjs index 4b75bd53..c7436c4d 100644 --- a/tools/cli-bootstrap/witness.test.mjs +++ b/tools/cli-bootstrap/witness.test.mjs @@ -413,20 +413,24 @@ test('output cap counts UTF-8 bytes and joins the killed child', async () => { assert.equal(result.signal, 'SIGKILL'); }); test( - 'an interrupted caller abandons a running control: its bounded child is killed, its fixture removed', + 'an interrupted caller abandons a running control: its bounded process group is killed, its fixture removed', { skip: process.platform === 'win32' }, async (t) => { const temporary = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-bootstrap-abandon-'))); t.after(() => rmSync(temporary, { recursive: true, force: true })); - const pidFile = join(temporary, 'child.pid'); + const pidFiles = ['child.pid', 'grandchild.pid'].map((name) => join(temporary, name)); const url = (name) => JSON.stringify(new URL(name, import.meta.url).href); - // The bounded child keeps writing into its fixture, as a running control does, and outlives the - // fixture's removal unless it is killed. - const writer = + // The bounded child and a grandchild in its process group keep writing into the fixture, as a + // running control's processes do, and outlive the fixture's removal unless they are killed. + const writer = (pidFile) => `const { writeFileSync } = require('node:fs');` + `const { join } = require('node:path');` + `writeFileSync(${JSON.stringify(pidFile)}, String(process.pid));` + `let n = 0; setInterval(() => { try { writeFileSync(join(process.env.TMPDIR, 'f' + n++), 'x'); } catch {} }, 5);`; + const bounded = + `require('node:child_process').spawn(process.execPath, ['-e', ${JSON.stringify( + writer(pidFiles[1]), + )}], { stdio: 'ignore' });` + writer(pidFiles[0]); const script = ` import { abandonFixtures, withFixture } from ${url('./child.mjs')}; import { spawnBounded } from ${url('./process.mjs')}; @@ -435,7 +439,7 @@ process.on('SIGINT', () => { process.exit(130); }); await withFixture((fixture) => - spawnBounded(['-e', ${JSON.stringify(writer)}], { env: { TMPDIR: fixture.temporary } }, { + spawnBounded(['-e', ${JSON.stringify(bounded)}], { env: { TMPDIR: fixture.temporary } }, { timeoutMs: 60_000, }), ); @@ -444,17 +448,23 @@ await withFixture((fixture) => env: { PATH: process.env.PATH, TMPDIR: temporary }, stdio: ['ignore', 'pipe', 'pipe'], }); - let child; + const alive = (pid) => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } + }; + const tree = []; try { - for ( - let waited = 0; - !existsSync(pidFile) || readFileSync(pidFile, 'utf8') === ''; - waited += 25 - ) { - assert.ok(waited < 20_000, 'the bounded child never started'); - await delay(25); + for (const file of pidFiles) { + for (let waited = 0; !existsSync(file) || readFileSync(file, 'utf8') === ''; waited += 25) { + assert.ok(waited < 20_000, 'the bounded control never started'); + await delay(25); + } + tree.push(Number(readFileSync(file, 'utf8'))); } - child = Number(readFileSync(pidFile, 'utf8')); assert.equal( readdirSync(temporary).filter((name) => name.startsWith('relavium-bootstrap-')).length, 1, @@ -464,16 +474,9 @@ await withFixture((fixture) => ); caller.kill('SIGINT'); assert.deepEqual(await exited, [130, null]); - const alive = () => { - try { - process.kill(child, 0); - return true; - } catch { - return false; - } - }; - for (let waited = 0; alive(); waited += 25) { - assert.ok(waited < 5_000, 'the bounded child outlived the abandoned control'); + // The whole process group goes, not only the process spawnBounded started. + for (let waited = 0; tree.some(alive); waited += 25) { + assert.ok(waited < 5_000, 'part of the bounded control outlived its abandonment'); await delay(25); } assert.deepEqual( @@ -482,13 +485,7 @@ await withFixture((fixture) => ); } finally { if (caller.exitCode === null && caller.signalCode === null) caller.kill('SIGKILL'); - if (child !== undefined) { - try { - process.kill(child, 'SIGKILL'); - } catch { - // Already stopped. - } - } + for (const pid of tree) if (alive(pid)) process.kill(pid, 'SIGKILL'); } }, ); diff --git a/tools/cli-package/README.md b/tools/cli-package/README.md index 88c483a7..1286db75 100644 --- a/tools/cli-package/README.md +++ b/tools/cli-package/README.md @@ -18,7 +18,8 @@ pnpm smoke:cli-package The producer (`produce.mjs`, run by `pack.mjs`) never builds in the developer's workspace. It copies the files Git would ship, by the repository's ignore rules and the developer's global excludes file (its Git reads no other home, global or system configuration), into a -fresh owner-only candidate tree, refusing if the original checkout's commit, status or set of +fresh owner-only candidate tree (owner-only where POSIX modes are observable; on Windows it +inherits the temporary directory's ACL, which is not verified), refusing if the original checkout's commit, status or set of files Git is told not to check changes during the copy (such a skip-worktree or assume-unchanged file makes the source `dirty`), and installs the locked closure there offline, frozen and without lifecycle scripts from the existing store. In that tree it emits the five @@ -43,7 +44,8 @@ The candidate's maps are built, validated and then removed with the rest of the private staging, also when the producer is interrupted (SIGINT, SIGTERM, SIGHUP), which stops the build first and empties the artifact directory, even after the last build child exited. An interrupted consumer stops its children, the bootstrap witness's included, and -removes its installation. `pnpm smoke:cli-package` forwards an interrupt to the producer or +removes its installation, the witness's fixtures and the installed smoke's working directory; the +installed CLI's detached native fixture targets end once the smoke worker that owns them is gone. `pnpm smoke:cli-package` forwards an interrupt to the producer or consumer it started instead of ending before them and starts no later stage after one; a producer killed outright can leave its owner-only root in the temporary directory. The receipt does **not** certify symbolication, a map manifest, encrypted custody or a production release candidate (Steps 9a.2–9d). No private map, diff --git a/tools/cli-package/candidate.mjs b/tools/cli-package/candidate.mjs index 780ab3f7..e35b1d81 100644 --- a/tools/cli-package/candidate.mjs +++ b/tools/cli-package/candidate.mjs @@ -199,7 +199,8 @@ export async function candidateFiles( /** * Copy each listed ordinary file byte-for-byte into the empty `destination`. Links, special files, * escaping or noncanonical paths and oversized trees refuse; only the executable bit is kept, as an - * exact 0755 or 0644 mode whatever the process umask. + * exact 0755 or 0644 mode under any umask that lets the owner write (one that removes the owner's + * write bit leaves the created directories unwritable, and the copy refuses before any build). * @param {{repository: string, destination: string, files: readonly string[], * limits?: {fileBytes: number, totalBytes: number}}} options `limits` defaults to `CANDIDATE_LIMITS` */ diff --git a/tools/cli-package/candidate.test.mjs b/tools/cli-package/candidate.test.mjs index c932e591..0e76d250 100644 --- a/tools/cli-package/candidate.test.mjs +++ b/tools/cli-package/candidate.test.mjs @@ -556,9 +556,11 @@ test('a candidate build leaves no build cache behind', () => { /** A source repository plus fake git and pnpm ports; the fake install creates the workspace links. */ function preparation({ - statusAfterCopy = '', + status = '', + statusAfterCopy = status, headAfterCopy = SHA, - uncheckedAfterCopy = '', + unchecked = '', + uncheckedAfterCopy = unchecked, excludes, } = {}) { const repository = fresh('source'); @@ -588,14 +590,14 @@ function preparation({ } if (key === `${prefix}status --porcelain=v1 -z --untracked-files=all`) { statuses += 1; - return { stdout: statuses === 1 ? '' : statusAfterCopy }; + return { stdout: statuses === 1 ? status : statusAfterCopy }; } if (key === `${prefix}ls-files -z --cached --others --exclude-standard`) return { stdout: files.join('\0') }; if (key === 'ls-files -z --deleted') return { stdout: '' }; if (key === 'ls-files -v -z') { flags += 1; - return { stdout: `H package.json\0${flags === 1 ? '' : uncheckedAfterCopy}` }; + return { stdout: `H package.json\0${flags === 1 ? unchecked : uncheckedAfterCopy}` }; } throw new Error(`unexpected git ${key}`); }; @@ -645,6 +647,25 @@ test('candidate preparation copies, re-checks identity, installs and verifies li ); }); +test('a modified or unchecked source makes the prepared identity dirty', async () => { + for (const source of [ + { status: ' M package.json\0' }, + { unchecked: 'S packages/core/package.json\0' }, + ]) { + const ports = preparation(source); + const identity = await prepareCandidate({ + repository: ports.repository, + candidate: ports.candidate, + environment: {}, + home: ports.home, + env: {}, + run: ports.run, + pnpm: ports.pnpm, + }); + assert.deepEqual(identity, { source: SHA, dirty: true, files: 6 }); + } +}); + test('a source change during the copy refuses before anything is installed', async () => { // A changed status, a commit that moved while the status stayed the same, or a file Git was told // not to check during the copy. diff --git a/tools/cli-package/check.test.mjs b/tools/cli-package/check.test.mjs index 0987a445..c4626f96 100644 --- a/tools/cli-package/check.test.mjs +++ b/tools/cli-package/check.test.mjs @@ -12,7 +12,7 @@ import { symlinkSync, writeFileSync, } from 'node:fs'; -import { tmpdir } from 'node:os'; +import { devNull, tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { after, test } from 'node:test'; import { setTimeout as delay } from 'node:timers/promises'; @@ -26,6 +26,10 @@ after(() => rmSync(scratch, { recursive: true, force: true })); let count = 0; const COPIED = Object.freeze([ + 'cli-bootstrap/child.mjs', + 'cli-bootstrap/policy.mjs', + 'cli-bootstrap/process.mjs', + 'cli-package/smoke/index.mjs', 'cli-package/check.mjs', 'cli-package/candidate.mjs', 'cli-package/consumer-lifetime.mjs', @@ -52,12 +56,14 @@ function writer(directory, pidFile) { /** The consumer stand-in: the real consumer lifetime around a `pnpm add`-like writer in its home. */ const CONSUMER = (markers) => ` -import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { completeConsumer } from './consumer-lifetime.mjs'; import { isolatedEnvironment, runCommand } from './process.mjs'; writeFileSync(${JSON.stringify(join(markers, 'consume-started'))}, String(process.pid)); +const reports = process.argv[3]; +mkdirSync(reports, { recursive: true }); const home = mkdtempSync(join(tmpdir(), 'relavium-package-consumer-')); const writer = (${writer.toString()})(home, ${JSON.stringify(join(markers, 'writer.pid'))}); await completeConsumer({ @@ -66,18 +72,49 @@ await completeConsumer({ env: isolatedEnvironment(home), timeoutMs: 60_000, }), - abandon: () => {}, + supportRoot: reports, cleanup: () => rmSync(home, { recursive: true, force: true, maxRetries: 3 }), cleanupFailed: () => {}, publish: () => {}, }); `; +/** A consumer stand-in that publishes the record `pnpm smoke:cli-package` summarizes. */ +const PUBLISHING_CONSUMER = (leg) => (markers) => ` +import { mkdirSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +writeFileSync(${JSON.stringify(join(markers, 'consume-started'))}, String(process.pid)); +const reports = process.argv[3]; +mkdirSync(reports, { recursive: true }); +writeFileSync(join(reports, 'installed-consumer.json'), ${JSON.stringify( + JSON.stringify({ + leg, + source: 'c'.repeat(40), + tarballSha256: 'd'.repeat(64), + node: 'v22.0.0', + platform: 'synthetic', + bootstrap: { controls: 7 }, + smoke: { cases: [{ id: 'one', passed: true }] }, + }), +)}); +`; + +/** A producer stand-in that records the store and excludes variables it received, then succeeds. */ +const RECORDING_PRODUCER = (markers) => ` +import { mkdirSync, writeFileSync } from 'node:fs'; +writeFileSync(${JSON.stringify(join(markers, 'pack-environment'))}, JSON.stringify({ + store: process.env.RELAVIUM_PACKAGE_STORE_DIR ?? null, + excludes: process.env.RELAVIUM_GIT_EXCLUDES_FILE ?? null, +})); +mkdirSync(process.argv[2]); +`; + /** - * Start the copied check with stand-ins for its producer and consumer. The store and the excludes - * file are named explicitly, so the check runs neither pnpm nor Git. + * Start the copied check with stand-ins for its producer and consumer. The store is named + * explicitly, so the check runs no pnpm; so is the excludes file unless `excludes` is false, when + * the check's Git lookup sees an empty home and no global or system configuration. */ -function check({ pack }) { +function check({ pack, consume = CONSUMER, excludes: named = true }) { count += 1; const root = join(scratch, `repository-${count}`); for (const path of COPIED) { @@ -95,14 +132,18 @@ function check({ pack }) { for (const directory of [markers, temporary, store]) mkdirSync(directory); const excludes = join(root, 'ignore'); writeFileSync(excludes, ''); + const home = join(root, 'home'); + mkdirSync(home); writeFileSync(join(root, 'tools', 'cli-package', 'pack.mjs'), pack(markers)); - writeFileSync(join(root, 'tools', 'cli-package', 'consume.mjs'), CONSUMER(markers)); + writeFileSync(join(root, 'tools', 'cli-package', 'consume.mjs'), consume(markers)); const child = spawn(process.execPath, [join(root, 'tools', 'cli-package', 'check.mjs')], { env: { PATH: process.env.PATH, TMPDIR: temporary, RELAVIUM_PACKAGE_STORE_DIR: store, - RELAVIUM_GIT_EXCLUDES_FILE: excludes, + ...(named + ? { RELAVIUM_GIT_EXCLUDES_FILE: excludes } + : { HOME: home, GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: devNull }), }, stdio: ['ignore', 'pipe', 'pipe'], }); @@ -111,7 +152,7 @@ function check({ pack }) { const exited = new Promise((resolve) => child.once('exit', (code, signal) => resolve({ code, signal, stdout })), ); - return { child, exited, markers, temporary }; + return { child, exited, markers, temporary, store, excludes }; } async function waitForFile(file) { @@ -228,3 +269,69 @@ setTimeout(() => process.exit(0), 500); } }, ); + +test('a successful producer and consumer give the summary and leave no check root', async () => { + const run = check({ pack: RECORDING_PRODUCER, consume: PUBLISHING_CONSUMER('local') }); + try { + const outcome = await run.exited; + assert.equal(outcome.signal, null); + assert.equal(outcome.code, 0); + assert.deepEqual(JSON.parse(outcome.stdout), { + scope: 'actual local host only; required CI owns four platform legs', + source: 'c'.repeat(40), + sha256: 'd'.repeat(64), + node: 'v22.0.0', + platform: 'synthetic', + controls: 7, + smokeCases: 1, + }); + // The isolated producer receives the store and the excludes file the check found. + assert.deepEqual(JSON.parse(readFileSync(join(run.markers, 'pack-environment'), 'utf8')), { + store: run.store, + excludes: run.excludes, + }); + assert.deepEqual(checkRoots(run.temporary), []); + } finally { + stopAll(run); + } +}); + +test('without a global excludes file the producer receives none', async () => { + const run = check({ + pack: RECORDING_PRODUCER, + consume: PUBLISHING_CONSUMER('local'), + excludes: false, + }); + try { + assert.equal((await run.exited).code, 0); + assert.deepEqual(JSON.parse(readFileSync(join(run.markers, 'pack-environment'), 'utf8')), { + store: run.store, + excludes: null, + }); + } finally { + stopAll(run); + } +}); + +for (const [label, options] of [ + ['a failed producer', { pack: () => 'process.exit(3);\n' }], + ['a failed consumer', { pack: RECORDING_PRODUCER, consume: () => 'process.exit(4);\n' }], + [ + 'a record from another leg', + { pack: RECORDING_PRODUCER, consume: PUBLISHING_CONSUMER('linux') }, + ], +]) + test(`${label} fails the check without a summary and leaves no check root`, async () => { + const run = check(options); + try { + const outcome = await run.exited; + assert.equal(outcome.signal, null); + assert.equal(outcome.code, 1); + assert.equal(outcome.stdout, ''); + if (label === 'a failed producer') + assert.equal(existsSync(join(run.markers, 'consume-started')), false); + assert.deepEqual(checkRoots(run.temporary), []); + } finally { + stopAll(run); + } + }); diff --git a/tools/cli-package/consume.mjs b/tools/cli-package/consume.mjs index bd5dd84c..930c1d1d 100644 --- a/tools/cli-package/consume.mjs +++ b/tools/cli-package/consume.mjs @@ -15,7 +15,6 @@ import { tmpdir } from 'node:os'; import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; import { fileURLToPath } from 'node:url'; import { runBootstrap } from '../cli-bootstrap/check.mjs'; -import { abandonFixtures } from '../cli-bootstrap/child.mjs'; import { inspectPackageArchive } from './archive.mjs'; import { originFromEnvironment, @@ -183,9 +182,9 @@ await completeConsumer({ validateConsumerRecord(record, receipt, { ...expected, allowLocal: true }); return record; }, - // An interrupt stops the bootstrap's own children and fixtures too; retries absorb a killed child - // that was still finishing a write. - abandon: abandonFixtures, + // An interrupt also abandons the bootstrap witness's fixtures and the installed smoke's worker roots + // in the report directory; retries absorb a killed child that was still finishing a write. + supportRoot: reportDirectory, cleanup: () => rmSync(home, { recursive: true, force: true, maxRetries: 3 }), cleanupFailed: () => writeSync(2, 'PACKED_CLI_SECONDARY_INSTALLATION_CLEANUP_FAILED\n'), publish(record) { diff --git a/tools/cli-package/consumer-lifetime.mjs b/tools/cli-package/consumer-lifetime.mjs index 8f9c9a43..5a251dab 100644 --- a/tools/cli-package/consumer-lifetime.mjs +++ b/tools/cli-package/consumer-lifetime.mjs @@ -1,18 +1,25 @@ import assert from 'node:assert/strict'; +import { isAbsolute } from 'node:path'; +import { abandonFixtures } from '../cli-bootstrap/child.mjs'; import { exitOnInterrupt } from './process.mjs'; +import { abandonInstalledSmoke } from './smoke/index.mjs'; /** * A success publication includes actual worker settlement and owned installation removal. An * interrupt (SIGINT, SIGTERM, SIGHUP) is the primary outcome: it stops every `runCommand` child tree, - * runs `abandon` for the children and directories `runCommand` does not own, removes the installation - * and exits with the signal's code, reporting a failed removal only through `cleanupFailed`. `abandon` - * and `cleanup` are therefore synchronous. + * then the bootstrap witness's bounded children, removes its live fixtures and the installed smoke's + * worker roots under `supportRoot` (the native fixture targets end with that worker), removes the + * installation and exits with the signal's code, reporting any failed removal only through + * `cleanupFailed`. `cleanup` is therefore synchronous. * @template T * @param {{exercise: () => Promise, cleanup: () => void, cleanupFailed: () => void, - * publish: (result: T) => unknown, abandon: () => void}} options + * publish: (result: T) => unknown, supportRoot: string}} options */ -export async function completeConsumer({ exercise, cleanup, cleanupFailed, publish, abandon }) { - assert.equal(typeof abandon, 'function', 'PACKED_CLI_CONSUMER_ABANDON'); +export async function completeConsumer({ exercise, cleanup, cleanupFailed, publish, supportRoot }) { + assert.ok( + typeof supportRoot === 'string' && isAbsolute(supportRoot), + 'PACKED_CLI_CONSUMER_SUPPORT_ROOT', + ); const secondary = () => { try { cleanupFailed(); @@ -21,12 +28,15 @@ export async function completeConsumer({ exercise, cleanup, cleanupFailed, publi } }; const interrupted = () => { - try { - abandon(); - cleanup(); - } catch { - secondary(); + let failed = false; + for (const step of [abandonFixtures, () => abandonInstalledSmoke(supportRoot), cleanup]) { + try { + step(); + } catch { + failed = true; + } } + if (failed) secondary(); }; return exitOnInterrupt(interrupted, async () => { let result; diff --git a/tools/cli-package/consumer-lifetime.test.mjs b/tools/cli-package/consumer-lifetime.test.mjs index d4e8306e..d4e8beba 100644 --- a/tools/cli-package/consumer-lifetime.test.mjs +++ b/tools/cli-package/consumer-lifetime.test.mjs @@ -14,12 +14,16 @@ import { spawn } from 'node:child_process'; import { createRequire } from 'node:module'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; -import { test } from 'node:test'; +import { after, test } from 'node:test'; import { setTimeout as delay } from 'node:timers/promises'; import { pathToFileURL } from 'node:url'; import { completeConsumer } from './consumer-lifetime.mjs'; import { REQUIRED_SMOKE_CASES } from './smoke/cases.mjs'; +// A support directory for the cases that never reach an interrupt. +const synthetic = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-consumer-support-'))); +after(() => rmSync(synthetic, { recursive: true, force: true })); + async function fixture( t, { wrongPid = false, exitCode = 0, nativeFailure = false, diagnosticSinkFailure = false } = {}, @@ -111,7 +115,7 @@ test('real addon worker closes before installation removal and success publicati // On Windows an in-process or still-running addon owner keeps this copy locked. rmSync(own.installation, { recursive: true, force: true }); }, - abandon: () => {}, + supportRoot: own.support, cleanupFailed: () => assert.fail('unexpected secondary cleanup failure'), publish: (report) => { assert.equal(existsSync(own.installation), false); @@ -146,7 +150,7 @@ for (const diagnosticSinkFailure of [false, true]) assert.equal(readFileSync(join(own.support, 'worker-exit'), 'utf8'), 'observed'); rmSync(own.installation, { recursive: true, force: true }); }, - abandon: () => {}, + supportRoot: own.support, cleanupFailed: () => assert.fail('unexpected secondary cleanup failure'), publish: () => { published = true; @@ -192,7 +196,7 @@ for (const [name, options, expected] of [ completeConsumer({ exercise: own.run, cleanup: () => rmSync(own.installation, { recursive: true, force: true }), - abandon: () => {}, + supportRoot: own.support, cleanupFailed: () => assert.fail('unexpected secondary cleanup failure'), publish: () => { published = true; @@ -214,7 +218,7 @@ test('failed required cleanup prevents publication of a successful exercise', as cleanup: () => { throw cleanupFailure; }, - abandon: () => {}, + supportRoot: synthetic, cleanupFailed: () => assert.fail('cleanup is primary here'), publish: () => { published = true; @@ -239,7 +243,7 @@ for (const primary of [new Error('synthetic worker refusal'), undefined, null]) cleaned = true; throw new Error('synthetic secondary cleanup refusal'); }, - abandon: () => {}, + supportRoot: synthetic, cleanupFailed: () => { diagnosed = true; throw new Error('synthetic failed diagnostic sink'); @@ -255,20 +259,23 @@ for (const primary of [new Error('synthetic worker refusal'), undefined, null]) assert.equal(published, false); }); -test('a consumer without an interrupt cleanup for the children runCommand does not own refuses', async () => { - let exercised = false; - await assert.rejects( - completeConsumer({ - exercise: async () => { - exercised = true; - }, - cleanup: () => {}, - cleanupFailed: () => {}, - publish: () => {}, - }), - /PACKED_CLI_CONSUMER_ABANDON/u, - ); - assert.equal(exercised, false); +test('a consumer without an absolute support directory refuses before it exercises', async () => { + for (const supportRoot of [undefined, 'relative/reports']) { + let exercised = false; + await assert.rejects( + completeConsumer({ + exercise: async () => { + exercised = true; + }, + cleanup: () => {}, + cleanupFailed: () => {}, + publish: () => {}, + supportRoot, + }), + /PACKED_CLI_CONSUMER_SUPPORT_ROOT/u, + ); + assert.equal(exercised, false); + } }); function alive(pid) { @@ -281,35 +288,79 @@ function alive(pid) { } /** - * A consumer process whose exercise is a long-lived `runCommand` child that keeps writing into the - * owned installation, as `pnpm add` does, and outlives the installation's removal unless it is killed; - * the test interrupts the consumer once the child is running. + * Code for a long-lived process that records its pid, then tries to write into the directory + * `directory` (a JavaScript expression) every 5 ms; it outlives that directory unless it is killed. + * With `grandchildFile`, it first starts a second such process in its own process group. + */ +function writer(directory, pidFile, grandchildFile) { + const own = + `require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, String(process.pid));` + + `let n = 0; setInterval(() => { try { require('node:fs').writeFileSync(` + + `require('node:path').join(${directory}, 'f' + n++), 'x'); } catch {} }, 5);`; + if (grandchildFile === undefined) return own; + return ( + `require('node:child_process').spawn(process.execPath, ['-e', ${JSON.stringify( + writer(directory, grandchildFile), + )}], { stdio: 'ignore' });` + own + ); +} + +async function waitForPid(file) { + for (let waited = 0; !existsSync(file) || readFileSync(file, 'utf8') === ''; waited += 25) { + assert.ok(waited < 20_000, `${file} never appeared`); + await delay(25); + } + return Number(readFileSync(file, 'utf8')); +} + +/** + * A consumer process interrupted mid-exercise: a `runCommand` child keeps writing into the owned + * installation, as `pnpm add` does, while a running bootstrap control's bounded child (with a + * grandchild in its process group) keeps writing into its fixture, and a smoke worker root, whose + * worker an interrupt kills before its own removal, waits in the support directory. */ async function interruptedConsumer(t, { cleanupFails = false } = {}) { const root = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-consumer-interrupt-'))); t.after(() => rmSync(root, { recursive: true, force: true })); const installation = join(root, 'installation'); - mkdirSync(installation); + const support = join(root, 'support'); + const temporary = join(root, 'tmp'); + for (const directory of [installation, support, temporary]) mkdirSync(directory); + const smoke = join(support, 'installed-smoke-planted'); + mkdirSync(smoke); + writeFileSync(join(smoke, 'guard.ndjson'), '{}\n'); const log = join(root, 'events'); - const pidFile = join(root, 'child.pid'); + const fixtureFile = join(root, 'fixture-root'); + const pids = ['install.pid', 'bounded.pid', 'grandchild.pid'].map((name) => join(root, name)); const url = (name) => JSON.stringify(new URL(name, import.meta.url).href); - const writer = - `const { writeFileSync } = require('node:fs');` + - `writeFileSync(${JSON.stringify(pidFile)}, String(process.pid));` + - `let n = 0; setInterval(() => { try { writeFileSync(require('node:path').join(${JSON.stringify(installation)}, 'f' + n++), 'x'); } catch {} }, 5);`; const script = ` -import { appendFileSync, rmSync } from 'node:fs'; +import { appendFileSync, existsSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { withFixture } from ${url('../cli-bootstrap/child.mjs')}; +import { spawnBounded } from ${url('../cli-bootstrap/process.mjs')}; import { completeConsumer } from ${url('./consumer-lifetime.mjs')}; import { isolatedEnvironment, runCommand } from ${url('./process.mjs')}; const record = (event) => appendFileSync(${JSON.stringify(log)}, event + '\\n'); +const state = (path) => (existsSync(path) ? 'live' : 'gone'); await completeConsumer({ - exercise: () => runCommand(process.execPath, ['-e', ${JSON.stringify(writer)}], { - env: isolatedEnvironment(${JSON.stringify(root)}), - timeoutMs: 60_000, - }), - abandon: () => record('abandon'), + supportRoot: ${JSON.stringify(support)}, + exercise: () => + Promise.all([ + runCommand(process.execPath, ['-e', ${JSON.stringify(writer(JSON.stringify(installation), pids[0]))}], { + env: isolatedEnvironment(${JSON.stringify(root)}), + timeoutMs: 60_000, + }), + withFixture((fixture) => { + writeFileSync(${JSON.stringify(fixtureFile)}, fixture.root); + return spawnBounded( + ['-e', ${JSON.stringify(writer('process.env.TMPDIR', pids[1], pids[2]))}], + { env: { TMPDIR: fixture.temporary } }, + { timeoutMs: 60_000 }, + ); + }), + ]), cleanup: () => { - record('cleanup'); + // Everything the interrupt abandons is gone before the installation's own removal. + record('cleanup fixture:' + state(readFileSync(${JSON.stringify(fixtureFile)}, 'utf8')) + ' smoke:' + state(${JSON.stringify(smoke)})); if (${cleanupFails}) throw new Error('synthetic cleanup refusal'); rmSync(${JSON.stringify(installation)}, { recursive: true, force: true, maxRetries: 3 }); }, @@ -318,42 +369,45 @@ await completeConsumer({ }); `; const consumer = spawn(process.execPath, ['--input-type=module', '-e', script], { + env: { PATH: process.env.PATH, TMPDIR: temporary }, stdio: ['ignore', 'pipe', 'pipe'], }); - let child; + const children = []; try { - for ( - let waited = 0; - !existsSync(pidFile) || readFileSync(pidFile, 'utf8') === ''; - waited += 25 - ) { - assert.ok(waited < 20_000, 'the exercise never started'); - await delay(25); - } - child = Number(readFileSync(pidFile, 'utf8')); + for (const file of pids) children.push(await waitForPid(file)); const exited = new Promise((resolve) => consumer.once('exit', (code, signal) => resolve([code, signal])), ); consumer.kill('SIGINT'); const [code, signal] = await exited; - return { code, signal, child, installation, events: readFileSync(log, 'utf8') }; + for (let waited = 0; children.some(alive); waited += 25) { + assert.ok(waited < 5_000, 'a child outlived the interrupted consumer'); + await delay(25); + } + return { + code, + signal, + installation, + smoke, + fixture: readFileSync(fixtureFile, 'utf8'), + events: readFileSync(log, 'utf8'), + }; } finally { if (consumer.exitCode === null && consumer.signalCode === null) consumer.kill('SIGKILL'); - if (child !== undefined && alive(child)) process.kill(child, 'SIGKILL'); + for (const pid of children) if (alive(pid)) process.kill(pid, 'SIGKILL'); } } test( - 'an interrupted consumer stops its children, abandons the rest, removes its installation and exits', + 'an interrupted consumer stops every child, abandons the bootstrap and smoke roots, removes its installation and exits', { skip: process.platform === 'win32' }, async (t) => { const outcome = await interruptedConsumer(t); assert.equal(outcome.signal, null); assert.equal(outcome.code, 130); - // The writer was stopped before the removal, which therefore leaves nothing behind. - assert.equal(alive(outcome.child), false); - assert.equal(existsSync(outcome.installation), false); - assert.equal(outcome.events, 'abandon\ncleanup\n'); + for (const path of [outcome.installation, outcome.smoke, outcome.fixture]) + assert.equal(existsSync(path), false, path); + assert.equal(outcome.events, 'cleanup fixture:gone smoke:gone\n'); }, ); @@ -364,6 +418,6 @@ test( const outcome = await interruptedConsumer(t, { cleanupFails: true }); assert.equal(outcome.signal, null); assert.equal(outcome.code, 130); - assert.equal(outcome.events, 'abandon\ncleanup\ncleanupFailed\n'); + assert.equal(outcome.events, 'cleanup fixture:gone smoke:gone\ncleanupFailed\n'); }, ); diff --git a/tools/cli-package/produce.mjs b/tools/cli-package/produce.mjs index dad134d6..c4012c55 100644 --- a/tools/cli-package/produce.mjs +++ b/tools/cli-package/produce.mjs @@ -20,8 +20,9 @@ import { emitFreshPackage } from './pipeline.mjs'; /** * Produce one package-test artifact from the original checkout `repository` into the empty - * `destination`. The candidate tree and its private staging are siblings in one owner-only temporary - * root that shares no path with the artifact directory or the checkout. That root and the build home + * `destination`. The candidate tree and its private staging are siblings in one temporary root, + * owner-only where POSIX modes are observable, that shares no path with the artifact directory or the + * checkout. That root and the build home * are removed whether production succeeds or fails. An interrupt (SIGINT, SIGTERM, SIGHUP) stops the * build's child trees, removes both directories and empties the artifact directory, which was empty * at the start, before the process exits with the signal's code — also when it arrives after the last diff --git a/tools/cli-package/produce.test.mjs b/tools/cli-package/produce.test.mjs index 696e4ad4..f22b8b23 100644 --- a/tools/cli-package/produce.test.mjs +++ b/tools/cli-package/produce.test.mjs @@ -37,7 +37,7 @@ const SHA = 'c'.repeat(40); * Fake ports that record what the producer composes. The fake emit writes into the candidate it is * given, runs the producer's seal wrapper and can plant a build-cache marker first. */ -function ports({ plantCache = false, failEmit = false } = {}) { +function ports({ plantCache = false, failEmit = false, dirty = false } = {}) { const seen = {}; return { seen, @@ -53,13 +53,22 @@ function ports({ plantCache = false, failEmit = false } = {}) { }, prepare: async (options) => { seen.prepare = options; - return { source: SHA, dirty: false, files: 7 }; + return { source: SHA, dirty, files: 7 }; }, emit: async (options) => { seen.emit = options; seen.stagingMode = statSync(options.staging).mode & 0o777; seen.rootMode = statSync(join(options.staging, '..')).mode & 0o777; mkdirSync(join(options.repository, 'apps', 'cli', 'dist'), { recursive: true }); + // Build state an engine package could carry; the emit removes it before each compile. + const core = join(options.repository, 'packages', 'core'); + mkdirSync(join(core, 'dist'), { recursive: true }); + writeFileSync(join(core, 'dist', 'stale.js'), 'stale\n'); + writeFileSync(join(core, 'tsconfig.build.tsbuildinfo'), '{}\n'); + await options.removeBuildState(core); + seen.buildStateLeft = ['dist', 'tsconfig.build.tsbuildinfo'].filter((name) => + existsSync(join(core, name)), + ); if (plantCache) writeFileSync(join(options.repository, '.turbo'), 'cache\n'); if (failEmit) throw new Error('synthetic emit failure'); writeFileSync(join(options.destination, 'relavium-0.1.1.tgz'), 'tarball'); @@ -132,6 +141,7 @@ test('the producer builds and seals only the owned candidate, never the original assert.equal(seen.emit.destination, destination); assert.equal(seen.emit.staging, join(candidate, '..', 'staging')); assert.equal(seen.noBuildCache, candidate); + assert.deepEqual(seen.buildStateLeft, []); assert.deepEqual(seen.seal, { artifactDirectory: destination, repository: candidate, @@ -151,6 +161,19 @@ test('the producer builds and seals only the owned candidate, never the original assert.deepEqual(readdirSync(destination), ['relavium-0.1.1.tgz']); }); +test('a dirty source identity reaches the seal and the returned identity', async () => { + const { seen, ports: fakes } = ports({ dirty: true }); + const result = await producePackage({ + repository: fresh('original-checkout'), + destination: join(fresh('artifact-parent'), 'artifact'), + environment: {}, + temporary: fresh('tmp'), + ports: fakes, + }); + assert.equal(seen.seal.dirty, true); + assert.equal(result.identity.dirty, true); +}); + test('a build cache left in the candidate refuses before the seal, and nothing private remains', async () => { const temporary = fresh('tmp'); const { seen, ports: fakes } = ports({ plantCache: true }); diff --git a/tools/cli-package/smoke/README.md b/tools/cli-package/smoke/README.md index 11fe2333..89d68f26 100644 --- a/tools/cli-package/smoke/README.md +++ b/tools/cli-package/smoke/README.md @@ -46,7 +46,9 @@ aggregate. Each result has `id`, `passed`, and evidence or an error. Overall - Replay a public offline `agent run --fixture` with `git_status` through the actual native process host. A private copied Node executable named `git`/`git.exe` runs the fixed extensionless fixture script, which creates only its recorded child. - Inject a synthetic error on the real target after readiness, then observe actual + The fixture's processes also end once the smoke worker that owns their case is + gone, so a consumer interrupted mid-case leaves no orphaned target; while the + worker lives, only the product's own cleanup stops them. Inject a synthetic error on the real target after readiness, then observe actual target and descendant closure. Delay delivery of real close events to distinguish an error/request from the owned native acknowledgement. POSIX additionally covers natural leader exit while a descendant holds inherited pipes. Windows runs the diff --git a/tools/cli-package/smoke/index.mjs b/tools/cli-package/smoke/index.mjs index 16ce07a3..42eb2a90 100644 --- a/tools/cli-package/smoke/index.mjs +++ b/tools/cli-package/smoke/index.mjs @@ -1,9 +1,27 @@ import assert from 'node:assert/strict'; +import { readdirSync, rmSync } from 'node:fs'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { readOrdinaryFile } from '../files.mjs'; import { isolatedEnvironment, runCommand } from '../process.mjs'; +/** The prefix of the worker's own temporary root under the support directory. */ +export const INSTALLED_SMOKE_ROOT_PREFIX = 'installed-smoke-'; + +/** + * For a caller that an interrupt ends while the worker runs: the worker is killed before its own + * removal, so remove every worker root under `supportRoot`. The native fixture targets the installed + * CLI started in their own process groups end on their own once that worker is gone. + * @param {string} supportRoot + */ +export function abandonInstalledSmoke(supportRoot) { + for (const name of readdirSync(supportRoot)) { + if (!name.startsWith(INSTALLED_SMOKE_ROOT_PREFIX)) continue; + // Retries absorb a killed process that was still finishing a write. + rmSync(join(supportRoot, name), { recursive: true, force: true, maxRetries: 3 }); + } +} + /** Join native module lifetime before the caller removes the owned installation. */ export async function runInstalledSmoke({ packageRoot, supportRoot, artifactSha256, source }) { const result = await runCommand( diff --git a/tools/cli-package/smoke/native-descendant.cjs b/tools/cli-package/smoke/native-descendant.cjs index 513c993d..1be988b4 100644 --- a/tools/cli-package/smoke/native-descendant.cjs +++ b/tools/cli-package/smoke/native-descendant.cjs @@ -2,7 +2,17 @@ const assert = require('node:assert/strict'); const { appendFileSync } = require('node:fs'); const { setInterval } = require('node:timers'); -assert.equal(process.argv.length, 3); +assert.equal(process.argv.length, 4); +const owner = Number(process.argv[3]); +assert.ok(Number.isSafeInteger(owner) && owner > 0); +// Like its parent, it never outlives the smoke worker that owns its case. +setInterval(() => { + try { + process.kill(owner, 0); + } catch (error) { + if (error.code === 'ESRCH') process.exit(0); + } +}, 100).unref(); appendFileSync( process.argv[2], JSON.stringify({ diff --git a/tools/cli-package/smoke/native-process.mjs b/tools/cli-package/smoke/native-process.mjs index f526bd6c..5eec4ec0 100644 --- a/tools/cli-package/smoke/native-process.mjs +++ b/tools/cli-package/smoke/native-process.mjs @@ -344,6 +344,8 @@ export async function assertInstalledNativeProcesses(context, { bootstrap = fals log, node: process.execPath, descendant: fixture('native-descendant.cjs'), + // This worker: the fixture's processes end once it is gone (an interrupted consumer). + owner: process.pid, ...(bootstrap ? { release } : {}), }), { mode: 0o600 }, diff --git a/tools/cli-package/smoke/native-process.test.mjs b/tools/cli-package/smoke/native-process.test.mjs index 5ceab75e..bb36fc4e 100644 --- a/tools/cli-package/smoke/native-process.test.mjs +++ b/tools/cli-package/smoke/native-process.test.mjs @@ -25,6 +25,7 @@ import { pendingNativeFixturePids, } from './native-process.mjs'; import { explicitEnvironment, ndjson, poll } from './process.mjs'; +import { setTimeout as sleep } from 'node:timers/promises'; function observedGroupRefusal() { return { @@ -206,6 +207,7 @@ test( log, node: descendantNode, descendant: fileURLToPath(new URL('./native-descendant.cjs', import.meta.url)), + owner: process.pid, }), ); const child = spawn(executable, ['status'], { @@ -313,6 +315,81 @@ test( }, ); +/** Whether `pid` still names a live process. */ +function alive(pid) { + try { + process.kill(pid, 0); + return true; + } catch (error) { + if (error.code === 'ESRCH') return false; + throw error; + } +} + +test( + 'the fixture target and its descendant end once the worker that owns their case is gone', + { timeout: 15000 }, + async () => { + const root = mkdtempSync(join(tmpdir(), 'relavium-status-owner-test-')); + const log = join(root, 'native.ndjson'); + // A stand-in for the smoke worker that an interrupted consumer kills. + const owner = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { + stdio: 'ignore', + }); + copyFileSync( + fileURLToPath(new URL('./native-status.cjs', import.meta.url)), + join(root, 'status'), + ); + writeFileSync( + join(root, 'native-fixture.json'), + JSON.stringify({ + mode: 'injected-error', + log, + node: process.execPath, + descendant: fileURLToPath(new URL('./native-descendant.cjs', import.meta.url)), + owner: owner.pid, + }), + ); + // Detached, as the installed CLI starts its native target, so nothing stops its group but the + // fixture's own watch of its owner. + const target = spawn(process.execPath, [join(root, 'status')], { + cwd: root, + detached: process.platform !== 'win32', + stdio: ['ignore', 'pipe', 'ignore'], + }); + let stdout = ''; + target.stdout.on('data', (chunk) => { + stdout += chunk.toString(); + }); + const pids = []; + try { + await poll( + () => stdout.includes('RELAVIUM_NATIVE_FIXTURE_READY\n'), + 5000, + 'Fixture descendant readiness missing', + ); + const descendant = ndjson(readFileSync(log, 'utf8')).find( + (item) => item.operation === 'native-fixture-descendant', + )?.pid; + assert.ok(Number.isSafeInteger(descendant) && descendant > 0); + pids.push(target.pid, descendant); + // Both stay while their owner lives. + await sleep(400); + assert.ok(pids.every(alive)); + owner.kill('SIGKILL'); + await poll( + () => !pids.some(alive), + 5000, + 'A fixture process outlived the worker that owns it', + ); + } finally { + for (const pid of pids) if (alive(pid)) process.kill(pid, 'SIGKILL'); + if (owner.exitCode === null && owner.signalCode === null) owner.kill('SIGKILL'); + rmSync(root, { recursive: true, force: true }); + } + }, +); + test('shared fixture runtime support reuses read-only identical bytes and refuses replacement content or links', () => { const root = mkdtempSync(join(tmpdir(), 'relavium-shared-runtime-test-')); const source = join(root, 'source.dylib'); diff --git a/tools/cli-package/smoke/native-status.cjs b/tools/cli-package/smoke/native-status.cjs index 07469538..59878492 100644 --- a/tools/cli-package/smoke/native-status.cjs +++ b/tools/cli-package/smoke/native-status.cjs @@ -19,12 +19,23 @@ assert.ok( 'bootstrap-output', ].includes(config.mode), ); +assert.ok(Number.isSafeInteger(config.owner) && config.owner > 0); +// The fixture never outlives the smoke worker that owns its case: an interrupted consumer kills that +// worker, and nothing else would stop a target in its own process group. Unreferenced, so it never +// keeps a fixture alive that would otherwise exit. +setInterval(() => { + try { + process.kill(config.owner, 0); + } catch (error) { + if (error.code === 'ESRCH') process.exit(0); + } +}, 100).unref(); record({ operation: 'native-fixture-root', pid: process.pid }); if (config.mode === 'exit-fallback') { process.stdout.write('RELAVIUM_NATIVE_FIXTURE_READY\n'); setInterval(() => {}, 1000); } else { - const descendant = spawn(config.node, [config.descendant, config.log], { + const descendant = spawn(config.node, [config.descendant, config.log, String(config.owner)], { shell: false, stdio: ['ignore', 'inherit', 'inherit', 'ipc'], }); diff --git a/tools/cli-package/smoke/worker.mjs b/tools/cli-package/smoke/worker.mjs index d54470c8..ad677e04 100644 --- a/tools/cli-package/smoke/worker.mjs +++ b/tools/cli-package/smoke/worker.mjs @@ -21,6 +21,7 @@ import { assertExit, explicitEnvironment, invoke, ndjson } from './process.mjs'; import { assertInstalledNativeProcesses, InstalledNativeControlError } from './native-process.mjs'; import { emitInstalledWorkerFailure } from './failure-diagnostic.mjs'; import { installedModulesOwner } from './installation.mjs'; +import { INSTALLED_SMOKE_ROOT_PREFIX } from './index.mjs'; const directory = dirname(fileURLToPath(import.meta.url)); const repository = resolve(directory, '../../..'); @@ -63,7 +64,7 @@ async function exerciseInstalledSmoke({ packageRoot, supportRoot, artifactSha256 const before = snapshot(packageRoot); const installedModules = installedModulesOwner(packageRoot); assert.ok(installedModules, 'Installed package must have an explicit node_modules owner'); - const root = mkdtempSync(join(supportRoot, 'installed-smoke-')); + const root = mkdtempSync(join(supportRoot, INSTALLED_SMOKE_ROOT_PREFIX)); const log = join(root, 'guard.ndjson'); const report = { schemaVersion: 1, From 3c6c5dce602efe28cfe58a2ebac53a3851c9e7e1 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Thu, 22 Oct 2026 12:49:39 +0300 Subject: [PATCH 27/97] fix(repo): address the fifth Step 9a.1 review round Round 5 (at d51a2437) reported seven verified findings, one medium; its plan, correctness and integration lenses were clean. - prepareCandidate's link verification is tested as composed: an install that resolves an engine package or the root dependency directory to the original checkout refuses after the install. - The candidate copy refuses a listed path whose parent directory below the checkout is a link, so a tracked directory replaced by an ignored link cannot bring in bytes from outside the checkout. - A core.excludesFile that is present but empty disables global excludes, as in Git, instead of falling back to the default file. - Settled children leaving both live-child sets, the exact `export {};` rule, each field of a facade's empty map, an interrupt the consumer outlives and the smoke-root prefix limit are tested. - packaging.md and the 9a.1 checkpoint record the corrections and round 5. Gates: lint:tools, typecheck:tools, every pnpm run ci step except the three control families (their captured sources are unchanged), with the real producer and consumer in smoke:cli-package, and the acceptance tools under CI=true GITHUB_ACTIONS=true; a mutation run of ten mutants over the corrections kills all of them. Refs: ADR-0111, ADR-0051 Co-Authored-By: Claude Opus 5.5 --- docs/reference/cli/packaging.md | 4 +- .../phase-2.5.5-hardening-and-remediation.md | 1 + tools/cli-bootstrap/witness.test.mjs | 17 +++- tools/cli-package/candidate.mjs | 29 ++++-- tools/cli-package/candidate.test.mjs | 98 ++++++++++++++++++- tools/cli-package/check.test.mjs | 23 +++++ tools/cli-package/consumer-lifetime.test.mjs | 11 +++ tools/cli-package/finalize.test.mjs | 27 +++-- tools/cli-package/process.test.mjs | 21 ++++ 9 files changed, 215 insertions(+), 16 deletions(-) diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 7101a80f..13340ef3 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -242,7 +242,9 @@ describes the implemented mechanism; independent acceptance is recorded in the plus untracked-but-not-ignored, minus deletions, by the repository's ignore rules and the developer's global excludes file, passed explicitly; that Git reads no home, global or system configuration) into a fresh owner-only tree, and refuses if any of the three - changed during the copy. + changed during the copy. A link, as a listed file or as any parent directory of one + below the checkout, refuses the copy, so a tracked directory replaced by an ignored link + cannot bring in bytes from outside the checkout. Ignored output, caches, dependencies and private notes never enter it. The locked closure is installed there offline, frozen and with `--ignore-scripts` from the existing content-addressed store, discovered with pnpm's store-location variables diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index fb25d0db..04773565 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -9585,6 +9585,7 @@ adversarially verified; the detailed reports stay local and Git-ignored. | 2 | `8ddb7ec2` | NONCLEAN (plan and integration lenses clean) | 12 (4 medium, 7 low, 1 nit; 9 distinct) | the producer removed its interrupt handlers before the final removal, so an interrupt during it ended the process with the private root half removed: the handlers now stay until everything private is gone, and a deterministic test interrupts that removal; SIGHUP, the handlers' removal and a whole build tree (a grandchild in the build's process group) are tested; path containment is decided by segments in one shared helper (`..private` is inside); private staging and the producer's placement compare the real paths and letter case the filesystem stores (`realpathSync.native`); the local `pnpm smoke:cli-package` forwards an interrupt to the producer and the consumer it started (`forwardInterrupts`); the finalizer creates its maps directory with an exact mode and its staged and public modes are tested under a umask that removes the owner's write bit; further finalizer and staged-closure refusals gained killing tests; the ADR-0051 note links ADR-0111 | | 3 | `184562bd` | NONCLEAN (security lens clean) | 11 (2 medium, 8 low, 1 nit) | round 2 still dropped an interrupt that arrived after the build's last child exited, during the seal or the final removal, because Node delivers a caught signal only when the event loop next polls and the handlers were removed first; round 2's test asserted exit 0 and so pinned the dropped interrupt. Every tool now keeps its handlers until the loop has polled once more, which takes two turns when the work ran from an I/O callback (one turn, the suggested fix, was measured to drop it), and the producer then empties the artifact directory and exits with the signal's code, tested from an I/O-callback tail. An interrupted consumer stops its own and the bootstrap witness's children and removes its installation and fixtures, and the local `pnpm smoke:cli-package`, now tested end to end, starts no later stage once an interrupt arrived. The build's isolated environment, the producer's native real path of the checkout and the prepack catch-all gained killing tests. Git reading the checkout uses no home, global or system configuration, a skip-worktree or assume-unchanged file makes the identity dirty, and an empty upstream mapping is accepted only for `export {};` | | 4 | `aa5c994a` | NONCLEAN (integration and critic lenses clean) | 10 (3 medium, 5 low, 2 nits) | an interrupted consumer killed the installed smoke's worker before its own removal, so the worker's directory (42 MB) stayed under the report directory and the native fixture target the installed CLI had started in its own process group ran on as an orphan: the fixture's processes now end once the smoke worker that owns their case is gone, and the consumer lifetime, given the report directory as a required `supportRoot`, abandons the bootstrap witness's children and fixtures and the smoke's worker roots itself before removing the installation. The local check's ordinary failures, its successful summary and its hand-off of the store and excludes variables, a dirty identity's path to the seal, the producer's removal of engine build state and the bounded children's process group gained killing tests. "Owner-only" is qualified where POSIX modes are not observable (Windows); the candidate copy's umask wording and the checkpoint's scope list were corrected | +| 5 | `d51a2437` | NONCLEAN (plan, correctness and integration lenses clean) | 7 (1 medium, 6 low) | `prepareCandidate`'s link verification had no composed test, so deleting the call passed: a preparation whose install resolves an engine package or the root dependency directory to the original checkout now refuses after the install. A tracked directory replaced by an ignored link let the copy read bytes from outside the checkout through the linked parent, so every parent directory below the checkout must now be an ordinary directory; a `core.excludesFile` that is present but empty disables Git's global excludes and no longer falls back to the default ignore file. Settled children leaving both live-child sets, the exact `export {};` rule, each field of a facade's empty map, an interrupt the consumer outlives and the smoke-root prefix limit gained killing tests | **Remaining gates.** The increment is not independently accepted. Two NEW consecutive clean complete cumulative rounds after the last material fix and a separate security sitting remain (the Acceptance diff --git a/tools/cli-bootstrap/witness.test.mjs b/tools/cli-bootstrap/witness.test.mjs index c7436c4d..8c5a9ff5 100644 --- a/tools/cli-bootstrap/witness.test.mjs +++ b/tools/cli-bootstrap/witness.test.mjs @@ -18,7 +18,7 @@ import { join, relative } from 'node:path'; import { pathToFileURL } from 'node:url'; import { withFixture, toEntryUrl } from './child.mjs'; import { assertWitness, importFailureCanary, servicePackages } from './policy.mjs'; -import { spawnBounded } from './process.mjs'; +import { spawnBounded, stopBoundedChildren } from './process.mjs'; import { BOOTSTRAP_CONTROL_COUNT, controlName, grammarCases, metaCases } from './inventory.mjs'; import ts from 'typescript'; @@ -489,3 +489,18 @@ await withFixture((fixture) => } }, ); +test( + 'a closed bounded child leaves nothing for an interrupted caller to kill', + { skip: process.platform === 'win32' }, + async (t) => { + // A closed child's process group may be recycled; abandoning must never signal it. + assert.equal((await spawnBounded(['-e', ''], {})).code, 0); + assert.equal( + (await spawnBounded(['-e', 'setInterval(() => {}, 1000)'], {}, { timeoutMs: 100 })).timedOut, + true, + ); + const kill = t.mock.method(process, 'kill', () => true); + stopBoundedChildren(); + assert.equal(kill.mock.callCount(), 0); + }, +); diff --git a/tools/cli-package/candidate.mjs b/tools/cli-package/candidate.mjs index e35b1d81..045c3108 100644 --- a/tools/cli-package/candidate.mjs +++ b/tools/cli-package/candidate.mjs @@ -65,7 +65,8 @@ const GIT_CONFIG_VARIABLES = Object.freeze([ * The developer's global Git excludes file, so that the candidate's isolated Git ignores what the * developer's own `git status` ignores: the explicit `RELAVIUM_GIT_EXCLUDES_FILE` when a caller * already isolated this process's environment, otherwise `core.excludesFile` as the developer's Git - * reads it, else Git's default `$XDG_CONFIG_HOME/git/ignore` or `$HOME/.config/git/ignore`; undefined + * reads it (present but empty: none, as Git then applies no global excludes), else, when the setting + * is absent, Git's default `$XDG_CONFIG_HOME/git/ignore` or `$HOME/.config/git/ignore`; undefined * when no such file exists. Only that location is read from the developer's configuration: every * candidate Git command runs without global or system configuration (`isolatedGitEnvironment`), so no * global hook, fsmonitor or other setting applies, wherever it is stored. @@ -82,7 +83,7 @@ export async function globalExcludesFile({ repository, environment, run = runCom for (const key of GIT_CONFIG_VARIABLES) { if (environment[key] !== undefined) env[key] = environment[key]; } - let configured = ''; + let configured; try { configured = ( await run('git', ['config', '--path', '--get', 'core.excludesFile'], { cwd: repository, env }) @@ -94,7 +95,9 @@ export async function globalExcludesFile({ repository, environment, run = runCom const xdg = environment.XDG_CONFIG_HOME; const home = environment.HOME ?? environment.USERPROFILE; let path; - if (configured !== '') path = resolve(repository, configured); + // A setting present but empty disables Git's global excludes; only an absent one has a default. + if (configured !== undefined) + path = configured === '' ? undefined : resolve(repository, configured); else if (xdg !== undefined && xdg !== '') path = join(xdg, 'git', 'ignore'); else if (home !== undefined && home !== '') path = join(home, '.config', 'git', 'ignore'); return path !== undefined && existsSync(path) && statSync(path).isFile() ? path : undefined; @@ -197,8 +200,9 @@ export async function candidateFiles( } /** - * Copy each listed ordinary file byte-for-byte into the empty `destination`. Links, special files, - * escaping or noncanonical paths and oversized trees refuse; only the executable bit is kept, as an + * Copy each listed ordinary file byte-for-byte into the empty `destination`. Links (also as a parent + * directory of a listed path), special files, escaping or noncanonical paths and oversized trees + * refuse; only the executable bit is kept, as an * exact 0755 or 0644 mode under any umask that lets the owner write (one that removes the owner's * write bit leaves the created directories unwritable, and the copy refuses before any build). * @param {{repository: string, destination: string, files: readonly string[], @@ -210,6 +214,7 @@ export function copyCandidateTree({ repository, destination, files, limits = CAN 'PACKED_CLI_CANDIDATE_ROOT', ); let total = 0; + const directories = new Set(); for (const path of files) { assert.ok( typeof path === 'string' && @@ -218,7 +223,19 @@ export function copyCandidateTree({ repository, destination, files, limits = CAN path.split('/').every((part) => part !== '' && part !== '.' && part !== '..'), 'PACKED_CLI_CANDIDATE_PATH', ); - const source = join(repository, ...path.split('/')); + const parts = path.split('/'); + // Every parent below the checkout must be an ordinary directory: Git lists the tracked children + // of a directory replaced by an (ignored) link, and reading through it would copy outside bytes. + for (let depth = 1; depth < parts.length; depth += 1) { + const parent = parts.slice(0, depth).join('/'); + if (directories.has(parent)) continue; + assert.ok( + lstatSync(join(repository, ...parts.slice(0, depth))).isDirectory(), + 'PACKED_CLI_CANDIDATE_PATH', + ); + directories.add(parent); + } + const source = join(repository, ...parts); const executable = (lstatSync(source).mode & 0o111) !== 0; const bytes = readOrdinaryFile(source, limits.fileBytes); total += bytes.length; diff --git a/tools/cli-package/candidate.test.mjs b/tools/cli-package/candidate.test.mjs index 0e76d250..17c482d2 100644 --- a/tools/cli-package/candidate.test.mjs +++ b/tools/cli-package/candidate.test.mjs @@ -6,6 +6,7 @@ import { readFileSync, readdirSync, realpathSync, + renameSync, rmSync, statSync, symlinkSync, @@ -213,6 +214,15 @@ test("the developer's global excludes file is located without the rest of the gl }), join(repository, 'local-ignore'), ); + // A setting present but empty disables Git's global excludes: the default file no longer applies. + assert.equal( + await globalExcludesFile({ + repository: '/repository', + environment: { HOME: home }, + run: answer('\n'), + }), + undefined, + ); // Any other Git failure refuses, and an explicit file from an isolating caller must exist. const broken = async () => { const error = new Error('PACKED_CLI_CHILD_FAILED: 128'); @@ -276,6 +286,19 @@ test('a real checkout copies nothing its developer ignores globally', async () = assert.equal((await sourceIdentity(repository)).dirty, true); }); +test('a real developer configuration with an empty core.excludesFile applies no global excludes', async () => { + const home = fresh('developer-home'); + write(join(home, '.gitconfig'), '[core]\n\texcludesFile =\n'); + write(join(home, '.config', 'git', 'ignore'), 'secret.local\n'); + assert.equal( + await globalExcludesFile({ + repository: fresh('repository'), + environment: { PATH: process.env.PATH, HOME: home, GIT_CONFIG_NOSYSTEM: '1' }, + }), + undefined, + ); +}); + /** A real repository with one committed file, `kept.ts`, created with Git's own configuration off. */ async function committedRepository(name) { const repository = fresh(name); @@ -315,6 +338,39 @@ test("no configuration found inside the checkout acts as Git's global configurat assert.equal((await sourceIdentity(repository)).dirty, true); }); +for (const ignoredBy of ['.gitignore', '.git/info/exclude']) + test( + `a tracked directory replaced by a link ignored through ${ignoredBy} refuses the copy`, + { skip: !posixModes }, + async () => { + const { repository, inRepository } = await committedRepository('linked-parent'); + write(join(repository, 'a', 'b'), 'tracked\n'); + await inRepository(['add', 'a/b']); + await inRepository([ + '-c', + 'user.name=t', + '-c', + 'user.email=t@example.invalid', + 'commit', + '--quiet', + '-m', + 'two', + ]); + // The directory moves outside the checkout and comes back as a link that Git ignores. + const outside = join(fresh('outside'), 'a'); + renameSync(join(repository, 'a'), outside); + write(join(outside, 'b'), 'outside bytes\n'); + symlinkSync(outside, join(repository, 'a'), linkKind); + write(join(repository, ...ignoredBy.split('/')), 'a\n'); + const files = await candidateFiles(repository); + assert.ok(files.includes('a/b')); + assert.throws( + () => copyCandidateTree({ repository, destination: fresh('copy'), files }), + /CANDIDATE_PATH/u, + ); + }, + ); + for (const flag of ['--skip-worktree', '--assume-unchanged']) test(`a tracked file marked ${flag} makes the identity dirty`, async () => { const { repository, inRepository } = await committedRepository('unchecked'); @@ -368,6 +424,19 @@ test('the candidate copy is byte-exact, keeps only the executable bit and refuse ); } if (posixModes) { + // A link as a parent directory refuses too, not only as the file itself. + const elsewhere = fresh('elsewhere'); + write(join(elsewhere, 'inner.ts'), 'outside\n'); + symlinkSync(elsewhere, join(repository, 'linked-directory'), linkKind); + assert.throws( + () => + copyCandidateTree({ + repository, + destination: fresh('copy'), + files: ['linked-directory/inner.ts'], + }), + /CANDIDATE_PATH/u, + ); symlinkSync(join(repository, 'package.json'), join(repository, 'linked.json')); assert.throws( () => copyCandidateTree({ repository, destination: fresh('copy'), files: ['linked.json'] }), @@ -562,6 +631,7 @@ function preparation({ unchecked = '', uncheckedAfterCopy = unchecked, excludes, + misLink, } = {}) { const repository = fresh('source'); write(join(repository, 'package.json'), '{}\n'); @@ -605,11 +675,16 @@ function preparation({ events.push(`pnpm ${args.join(' ')}`); if (args[0] === 'store') return { stdout: `${store}\n` }; assert.equal(options.cwd, candidate); - mkdirSync(join(candidate, 'node_modules'), { recursive: true }); + // `misLink` makes the install resolve one engine package, or the root dependency directory, to + // the original checkout instead of the candidate. + if (misLink === 'root') + symlinkSync(fresh('borrowed-modules'), join(candidate, 'node_modules'), linkKind); + else mkdirSync(join(candidate, 'node_modules'), { recursive: true }); const scope = join(candidate, 'apps', 'cli', 'node_modules', '@relavium'); mkdirSync(scope, { recursive: true }); for (const name of ['shared', 'llm', 'core', 'db', 'mcp']) { - symlinkSync(join(candidate, 'packages', name), join(scope, name), linkKind); + const owner = misLink === 'engine' && name === 'core' ? repository : candidate; + symlinkSync(join(owner, 'packages', name), join(scope, name), linkKind); } return { stdout: '' }; }; @@ -666,6 +741,25 @@ test('a modified or unchecked source makes the prepared identity dirty', async ( } }); +test('an installed link outside the candidate refuses after the install', async () => { + for (const misLink of ['engine', 'root']) { + const ports = preparation({ misLink }); + await assert.rejects( + prepareCandidate({ + repository: ports.repository, + candidate: ports.candidate, + environment: {}, + home: ports.home, + env: {}, + run: ports.run, + pnpm: ports.pnpm, + }), + /CANDIDATE_LINK/u, + ); + assert.match(ports.events.at(-1), /^pnpm install /u); + } +}); + test('a source change during the copy refuses before anything is installed', async () => { // A changed status, a commit that moved while the status stayed the same, or a file Git was told // not to check during the copy. diff --git a/tools/cli-package/check.test.mjs b/tools/cli-package/check.test.mjs index c4626f96..6d29e57a 100644 --- a/tools/cli-package/check.test.mjs +++ b/tools/cli-package/check.test.mjs @@ -335,3 +335,26 @@ for (const [label, options] of [ stopAll(run); } }); + +test( + 'an interrupt the consumer outlives still gives no summary', + { skip: process.platform === 'win32' }, + async () => { + // The consumer signals the check, ignores the interrupt forwarded back to it and publishes. + const outliving = (markers) => + PUBLISHING_CONSUMER('local')(markers) + + `process.on('SIGINT', () => writeFileSync(${JSON.stringify(join(markers, 'consume-forwarded'))}, 'SIGINT'));\n` + + `process.kill(process.ppid, 'SIGINT');\nsetTimeout(() => process.exit(0), 500);\n`; + const run = check({ pack: RECORDING_PRODUCER, consume: outliving }); + try { + const outcome = await run.exited; + assert.equal(outcome.signal, null); + assert.equal(outcome.code, 130); + assert.equal(outcome.stdout, ''); + assert.equal(readFileSync(join(run.markers, 'consume-forwarded'), 'utf8'), 'SIGINT'); + assert.deepEqual(checkRoots(run.temporary), []); + } finally { + stopAll(run); + } + }, +); diff --git a/tools/cli-package/consumer-lifetime.test.mjs b/tools/cli-package/consumer-lifetime.test.mjs index d4e8beba..792b1a74 100644 --- a/tools/cli-package/consumer-lifetime.test.mjs +++ b/tools/cli-package/consumer-lifetime.test.mjs @@ -6,6 +6,7 @@ import { mkdirSync, mkdtempSync, readFileSync, + readdirSync, realpathSync, rmSync, writeFileSync, @@ -329,6 +330,12 @@ async function interruptedConsumer(t, { cleanupFails = false } = {}) { const smoke = join(support, 'installed-smoke-planted'); mkdirSync(smoke); writeFileSync(join(smoke, 'guard.ndjson'), '{}\n'); + // Report entries that are not worker roots, a near miss included, which the interrupt keeps. + const kept = ['install.log', 'installed-smoke.json', 'installed-smokeX', 'bootstrap']; + writeFileSync(join(support, 'install.log'), 'log\n'); + writeFileSync(join(support, 'installed-smoke.json'), '{}\n'); + writeFileSync(join(support, 'installed-smokeX'), 'near miss\n'); + mkdirSync(join(support, 'bootstrap')); const log = join(root, 'events'); const fixtureFile = join(root, 'fixture-root'); const pids = ['install.pid', 'bounded.pid', 'grandchild.pid'].map((name) => join(root, name)); @@ -391,6 +398,8 @@ await completeConsumer({ smoke, fixture: readFileSync(fixtureFile, 'utf8'), events: readFileSync(log, 'utf8'), + support: readdirSync(support).sort(), + kept: kept.sort(), }; } finally { if (consumer.exitCode === null && consumer.signalCode === null) consumer.kill('SIGKILL'); @@ -407,6 +416,8 @@ test( assert.equal(outcome.code, 130); for (const path of [outcome.installation, outcome.smoke, outcome.fixture]) assert.equal(existsSync(path), false, path); + // Only the worker roots go: every other report entry stays. + assert.deepEqual(outcome.support, outcome.kept); assert.equal(outcome.events, 'cleanup fixture:gone smoke:gone\n'); }, ); diff --git a/tools/cli-package/finalize.test.mjs b/tools/cli-package/finalize.test.mjs index 80e76f17..a3c376d8 100644 --- a/tools/cli-package/finalize.test.mjs +++ b/tools/cli-package/finalize.test.mjs @@ -322,6 +322,17 @@ test('refuses an unparsable upstream map and an inexact upstream map link', asyn } }); +test('accepts empty upstream mappings only for exactly the JS tsc emits for erased types', async () => { + // `export {};` followed by executable code is not a module whose types all erase. + const candidate = await fixture(); + writeFileSync( + join(candidate.core, 'dist', 'index.js'), + `export {};\nconsole.log(1);\n${mapTrailer('index.js')}`, + ); + editJson(join(candidate.core, 'dist', 'index.js.map'), (map) => (map.mappings = '')); + refusesBeforeWriting(candidate, 'UPSTREAM_MAP_UNMAPPED'); +}); + test('refuses final maps that embed sources, lose coverage or carry unreviewed fields', async () => { refusesBeforeWriting(await fixture({ sourcesContent: true }), 'EMBEDDED_SOURCES'); for (const [change, code] of [ @@ -347,12 +358,16 @@ test('refuses final maps that embed sources, lose coverage or carry unreviewed f editJson(join(candidate.dist, 'index.js.map'), change); refusesBeforeWriting(candidate, code); } - const candidate = await fixture(); - editJson(join(candidate.dist, `${facadeNames(candidate)[0]}.map`), (map) => { - map.sources = ['../src/facade.ts']; - map.mappings = 'AAAA'; - }); - refusesBeforeWriting(candidate, 'MAP_SHAPE'); + // A facade's map stays empty in every field: sources, mappings and names each refuse alone. + for (const change of [ + (map) => (map.sources = ['../src/facade.ts']), + (map) => (map.mappings = 'AAAA'), + (map) => (map.names = ['x']), + ]) { + const candidate = await fixture(); + editJson(join(candidate.dist, `${facadeNames(candidate)[0]}.map`), change); + refusesBeforeWriting(candidate, 'MAP_SHAPE'); + } }); test('removes only the exact terminal trailer that tsup appended after esbuild sized the output', async () => { diff --git a/tools/cli-package/process.test.mjs b/tools/cli-package/process.test.mjs index 496100a6..5cdba77e 100644 --- a/tools/cli-package/process.test.mjs +++ b/tools/cli-package/process.test.mjs @@ -21,6 +21,7 @@ import { resolvePnpm, runCommand, runPnpm, + stopLiveCommands, } from './process.mjs'; test('Git reading the original checkout gets no home, global or system configuration', () => { @@ -284,3 +285,23 @@ console.log('not interrupted'); assert.equal(stdout, ''); }, ); + +test( + 'a settled child leaves nothing for an interrupt to kill', + { skip: process.platform === 'win32' }, + async (t) => { + // A finished child's process group may be recycled; an interrupt must never signal it. + await runCommand(process.execPath, ['-e', ''], { timeoutMs: 20_000 }); + await assert.rejects( + runCommand(process.execPath, ['-e', 'process.exit(3)'], { timeoutMs: 20_000 }), + /PACKED_CLI_CHILD_FAILED/u, + ); + await assert.rejects( + runCommand(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { timeoutMs: 200 }), + /PACKED_CLI_CHILD_TIMEOUT/u, + ); + const kill = t.mock.method(process, 'kill', () => true); + stopLiveCommands(); + assert.equal(kill.mock.callCount(), 0); + }, +); From 0e59f5ffd383416f51d8584454e12edaaf843c77 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Thu, 22 Oct 2026 13:55:46 +0300 Subject: [PATCH 28/97] fix(repo): address the sixth Step 9a.1 review round MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 6 (at 3c6c5dce) reported six verified findings (four distinct), one medium; its plan and correctness lenses were clean. - A test pins that a leader-exit native descendant watches the smoke worker, not its exited parent, so the product's group reap stays what the installed control proves. - The staging directory's ownership check has a killing test. - Every wait for the local check's exit in check.test.mjs is bounded. - Five live status statements that still said Wave 3 Steps 9–19 had not started now say that Step 9a.1 is under review; the register records round 6. Gates: lint:tools, typecheck:tools, every pnpm run ci step except the three control families (their captured sources are unchanged), with the real producer and consumer in smoke:cli-package, and the acceptance tools under CI=true GITHUB_ACTIONS=true; a mutation run of three mutants over the corrections kills all of them. Refs: ADR-0111, ADR-0051 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 5 +- CLAUDE.md | 5 +- docs/roadmap/current.md | 7 +-- docs/roadmap/deferred-tasks.md | 2 +- .../phase-2.5.5-hardening-and-remediation.md | 1 + tools/cli-package/check.test.mjs | 23 ++++++--- tools/cli-package/finalize.test.mjs | 10 +++- .../cli-package/smoke/native-process.test.mjs | 50 +++++++++++++++++++ 8 files changed, 87 insertions(+), 16 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b2f30f9b..0ec1e1a1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -116,8 +116,9 @@ explicit; source CI does not certify later documentation commits. The [acceptance register](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 -itself is NOT complete: Steps 9–19 have NOT STARTED and whole-wave acceptance remains -Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) +itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now +under review below; Steps 9a.2–19 have not started) and whole-wave acceptance remains Step 19. +The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification only and was then independently reviewed (four rounds and a security sitting, with follow-up diff --git a/CLAUDE.md b/CLAUDE.md index 8817e9e3..9d84995f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -205,8 +205,9 @@ explicit; source CI does not certify later documentation commits. The [acceptance register](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 -itself is NOT complete: Steps 9–19 have NOT STARTED and whole-wave acceptance remains -Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) +itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now +under review below; Steps 9a.2–19 have not started) and whole-wave acceptance remains Step 19. +The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification only and was then independently reviewed (four rounds and a security sitting, with follow-up diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index 4224a310..7edacc35 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -137,8 +137,9 @@ The [deferred register](deferred-tasks.md) retains its existing limits; closure ## Wave 3 Steps 1–8 merged — PR #95 **Wave 3 Steps 1–8 merged to `main` on 2026-10-18 as [PR #95](https://github.com/HodeTech/Relavium/pull/95), -merge commit `6d2a7372` (22:44 UTC). Wave 3 itself is not complete: Steps 9–19 have NOT STARTED and -whole-wave acceptance remains Step 19.** The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) +merge commit `6d2a7372` (22:44 UTC). Wave 3 itself is not complete: Steps 9–19 had not started at +that merge (Step 9a.1 is now under review; Steps 9a.2–19 have not started) and whole-wave acceptance +remains Step 19.** The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) joins the completed scope, every review attribution, the independent review's dispositions and the publication state observed at merge. It was the maintainer-authorized, scoped publication checkpoint. @@ -1140,7 +1141,7 @@ an ESLint selector rather than a list. ### Wave 3 — Clear the ground -**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Steps 9–19 NOT STARTED — 2026-10-19.** +**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Step 9a.1 implemented on `development` and under review, Steps 9a.2–19 NOT STARTED — 2026-10-22.** ADR-0110–0113, their linked contracts and the nineteen-increment plan were approved on 2026-10-16. [Steps 1–8 were independently accepted](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) on `development`, with whole Step 8 reviewed at `c83c8575`, and published as the scoped checkpoint diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index 1437bca2..2ecad1f1 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -2,7 +2,7 @@ > Status: Living -> Last updated: 2026-10-22 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Steps 9–19 not started; the PR95 increment is closed on `development` under the maintainer's 2026-10-21 round cap). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's edge cases with the maintainer's 2026-10-21 acceptance bar and round cap and the 2026-10-22 rulings. Existing approved deferrals remain. +> Last updated: 2026-10-22 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Step 9a.1 is implemented on `development` and under review, Steps 9a.2–19 not started; the PR95 increment is closed on `development` under the maintainer's 2026-10-21 round cap). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's edge cases with the maintainer's 2026-10-21 acceptance bar and round cap and the 2026-10-22 rulings. Existing approved deferrals remain. > The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) records completed prerequisites; full M/N lifecycle and other scheduled work remain with their owners. > > Earlier triage: the Phase-2.6 rewrite triaged every open item, the 2026-07-19 full-project diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 04773565..485237d6 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -9586,6 +9586,7 @@ adversarially verified; the detailed reports stay local and Git-ignored. | 3 | `184562bd` | NONCLEAN (security lens clean) | 11 (2 medium, 8 low, 1 nit) | round 2 still dropped an interrupt that arrived after the build's last child exited, during the seal or the final removal, because Node delivers a caught signal only when the event loop next polls and the handlers were removed first; round 2's test asserted exit 0 and so pinned the dropped interrupt. Every tool now keeps its handlers until the loop has polled once more, which takes two turns when the work ran from an I/O callback (one turn, the suggested fix, was measured to drop it), and the producer then empties the artifact directory and exits with the signal's code, tested from an I/O-callback tail. An interrupted consumer stops its own and the bootstrap witness's children and removes its installation and fixtures, and the local `pnpm smoke:cli-package`, now tested end to end, starts no later stage once an interrupt arrived. The build's isolated environment, the producer's native real path of the checkout and the prepack catch-all gained killing tests. Git reading the checkout uses no home, global or system configuration, a skip-worktree or assume-unchanged file makes the identity dirty, and an empty upstream mapping is accepted only for `export {};` | | 4 | `aa5c994a` | NONCLEAN (integration and critic lenses clean) | 10 (3 medium, 5 low, 2 nits) | an interrupted consumer killed the installed smoke's worker before its own removal, so the worker's directory (42 MB) stayed under the report directory and the native fixture target the installed CLI had started in its own process group ran on as an orphan: the fixture's processes now end once the smoke worker that owns their case is gone, and the consumer lifetime, given the report directory as a required `supportRoot`, abandons the bootstrap witness's children and fixtures and the smoke's worker roots itself before removing the installation. The local check's ordinary failures, its successful summary and its hand-off of the store and excludes variables, a dirty identity's path to the seal, the producer's removal of engine build state and the bounded children's process group gained killing tests. "Owner-only" is qualified where POSIX modes are not observable (Windows); the candidate copy's umask wording and the checkpoint's scope list were corrected | | 5 | `d51a2437` | NONCLEAN (plan, correctness and integration lenses clean) | 7 (1 medium, 6 low) | `prepareCandidate`'s link verification had no composed test, so deleting the call passed: a preparation whose install resolves an engine package or the root dependency directory to the original checkout now refuses after the install. A tracked directory replaced by an ignored link let the copy read bytes from outside the checkout through the linked parent, so every parent directory below the checkout must now be an ordinary directory; a `core.excludesFile` that is present but empty disables Git's global excludes and no longer falls back to the default ignore file. Settled children leaving both live-child sets, the exact `export {};` rule, each field of a facade's empty map, an interrupt the consumer outlives and the smoke-root prefix limit gained killing tests | +| 6 | `3c6c5dce` | NONCLEAN (plan and correctness lenses clean) | 6 (1 medium, 4 low, 1 partial nit; 4 distinct) | nothing pinned that a leader-exit descendant watches the smoke worker rather than its own parent, so a descendant watching its exited parent would have ended on its own and left the product's group reap unproven: a test now keeps the descendant alive past its parent's exit until the owner is killed. The staging directory's ownership check gained a killing test (a different `process.getuid`); every wait for the local check's exit is bounded; and five live status statements that still said Steps 9–19 had not started now say that Step 9a.1 is under review | **Remaining gates.** The increment is not independently accepted. Two NEW consecutive clean complete cumulative rounds after the last material fix and a separate security sitting remain (the Acceptance diff --git a/tools/cli-package/check.test.mjs b/tools/cli-package/check.test.mjs index 6d29e57a..b5b940e5 100644 --- a/tools/cli-package/check.test.mjs +++ b/tools/cli-package/check.test.mjs @@ -155,6 +155,15 @@ function check({ pack, consume = CONSUMER, excludes: named = true }) { return { child, exited, markers, temporary, store, excludes }; } +/** The check's exit, bounded so that a regressed check fails its test instead of hanging it. */ +async function exitOf(run) { + // Unreferenced, so a pending bound never keeps the test process alive after the check exits. + const bound = delay(60_000, undefined, { ref: false }).then(() => undefined); + const outcome = await Promise.race([run.exited, bound]); + assert.ok(outcome !== undefined, 'the check did not exit within 60 s'); + return outcome; +} + async function waitForFile(file) { for (let waited = 0; !existsSync(file) || readFileSync(file, 'utf8') === ''; waited += 25) { assert.ok(waited < 20_000, `${file} never appeared`); @@ -200,7 +209,7 @@ test( pid = Number(await waitForFile(join(run.markers, 'writer.pid'))); assert.equal(checkRoots(run.temporary).length, 1); run.child.kill('SIGINT'); - const outcome = await run.exited; + const outcome = await exitOf(run); assert.equal(outcome.signal, null); assert.equal(outcome.code, 130); assert.equal(outcome.stdout, ''); @@ -231,7 +240,7 @@ setInterval(() => {}, 1000); try { await waitForFile(join(run.markers, 'pack.pid')); run.child.kill('SIGINT'); - const outcome = await run.exited; + const outcome = await exitOf(run); assert.equal(outcome.signal, null); assert.equal(outcome.code, 130); assert.equal(readFileSync(join(run.markers, 'pack-interrupted'), 'utf8'), 'SIGINT'); @@ -258,7 +267,7 @@ setTimeout(() => process.exit(0), 500); `, }); try { - const outcome = await run.exited; + const outcome = await exitOf(run); assert.equal(outcome.signal, null); assert.equal(outcome.code, 130); assert.equal(readFileSync(join(run.markers, 'pack-forwarded'), 'utf8'), 'SIGINT'); @@ -273,7 +282,7 @@ setTimeout(() => process.exit(0), 500); test('a successful producer and consumer give the summary and leave no check root', async () => { const run = check({ pack: RECORDING_PRODUCER, consume: PUBLISHING_CONSUMER('local') }); try { - const outcome = await run.exited; + const outcome = await exitOf(run); assert.equal(outcome.signal, null); assert.equal(outcome.code, 0); assert.deepEqual(JSON.parse(outcome.stdout), { @@ -303,7 +312,7 @@ test('without a global excludes file the producer receives none', async () => { excludes: false, }); try { - assert.equal((await run.exited).code, 0); + assert.equal((await exitOf(run)).code, 0); assert.deepEqual(JSON.parse(readFileSync(join(run.markers, 'pack-environment'), 'utf8')), { store: run.store, excludes: null, @@ -324,7 +333,7 @@ for (const [label, options] of [ test(`${label} fails the check without a summary and leaves no check root`, async () => { const run = check(options); try { - const outcome = await run.exited; + const outcome = await exitOf(run); assert.equal(outcome.signal, null); assert.equal(outcome.code, 1); assert.equal(outcome.stdout, ''); @@ -347,7 +356,7 @@ test( `process.kill(process.ppid, 'SIGINT');\nsetTimeout(() => process.exit(0), 500);\n`; const run = check({ pack: RECORDING_PRODUCER, consume: outliving }); try { - const outcome = await run.exited; + const outcome = await exitOf(run); assert.equal(outcome.signal, null); assert.equal(outcome.code, 130); assert.equal(outcome.stdout, ''); diff --git a/tools/cli-package/finalize.test.mjs b/tools/cli-package/finalize.test.mjs index a3c376d8..27d6063e 100644 --- a/tools/cli-package/finalize.test.mjs +++ b/tools/cli-package/finalize.test.mjs @@ -462,7 +462,7 @@ test('refuses an orphan map entry and validates the upstream map of a zero-byte refusesBeforeWriting(zero, 'UPSTREAM_MAP_SHAPE'); }); -test('requires an empty private staging directory outside the candidate', async () => { +test('requires an empty private staging directory outside the candidate', async (t) => { const candidate = await fixture(); for (const [staging, code] of [ ['relative-staging', 'STAGING_PATH'], @@ -482,6 +482,14 @@ test('requires an empty private staging directory outside the candidate', async chmodSync(candidate.staging, 0o755); refusesBeforeWriting(candidate, 'STAGING_NOT_PRIVATE'); chmodSync(candidate.staging, 0o700); + // An owner-only directory another account owns refuses too, before anything is written. + const realUid = process.getuid(); + const uid = t.mock.method(process, 'getuid', () => realUid + 1); + try { + refusesBeforeWriting(candidate, 'STAGING_NOT_PRIVATE'); + } finally { + uid.mock.restore(); + } const link = join(scratch, `staging-link-${count}`); symlinkSync(candidate.staging, link); assert.throws( diff --git a/tools/cli-package/smoke/native-process.test.mjs b/tools/cli-package/smoke/native-process.test.mjs index bb36fc4e..7ea7c9a3 100644 --- a/tools/cli-package/smoke/native-process.test.mjs +++ b/tools/cli-package/smoke/native-process.test.mjs @@ -390,6 +390,56 @@ test( }, ); +test( + 'a leader-exit descendant outlives its exited parent until the worker that owns it is gone', + { timeout: 15000, skip: process.platform === 'win32' }, + async () => { + // Leader-exit mode reparents the descendant by design; it must watch the worker, never its parent, + // or it would end on its own and the product's group reap would go unproven. + const root = mkdtempSync(join(tmpdir(), 'relavium-status-leader-test-')); + const log = join(root, 'native.ndjson'); + const owner = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { + stdio: 'ignore', + }); + copyFileSync( + fileURLToPath(new URL('./native-status.cjs', import.meta.url)), + join(root, 'status'), + ); + writeFileSync( + join(root, 'native-fixture.json'), + JSON.stringify({ + mode: 'leader-exit', + log, + node: process.execPath, + descendant: fileURLToPath(new URL('./native-descendant.cjs', import.meta.url)), + owner: owner.pid, + }), + ); + const target = spawn(process.execPath, [join(root, 'status')], { + cwd: root, + detached: true, + stdio: ['ignore', 'ignore', 'ignore'], + }); + const leaderExit = new Promise((resolve) => target.once('exit', (code) => resolve(code))); + let descendant; + try { + assert.equal(await leaderExit, 0); + descendant = ndjson(readFileSync(log, 'utf8')).find( + (item) => item.operation === 'native-fixture-descendant', + )?.pid; + assert.ok(Number.isSafeInteger(descendant) && descendant > 0); + await sleep(600); + assert.equal(alive(descendant), true, 'the descendant ended with its parent, not its owner'); + owner.kill('SIGKILL'); + await poll(() => !alive(descendant), 5000, 'The descendant outlived the worker that owns it'); + } finally { + if (descendant !== undefined && alive(descendant)) process.kill(descendant, 'SIGKILL'); + if (owner.exitCode === null && owner.signalCode === null) owner.kill('SIGKILL'); + rmSync(root, { recursive: true, force: true }); + } + }, +); + test('shared fixture runtime support reuses read-only identical bytes and refuses replacement content or links', () => { const root = mkdtempSync(join(tmpdir(), 'relavium-shared-runtime-test-')); const source = join(root, 'source.dylib'); From c691ef9fdfc6a7e55fc695e1df3cbeb4d84fd60c Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Thu, 22 Oct 2026 14:58:49 +0300 Subject: [PATCH 29/97] fix(repo): address the seventh Step 9a.1 review round Round 7 (at 0e59f5ff) reported three verified low findings; its plan, security, correctness and integration lenses were clean. - Git reading the original checkout ignores replace refs (GIT_NO_REPLACE_OBJECTS), so a replace ref on HEAD can no longer make a substituted tree look clean to the source identity; a real-repository test pins it. - The linked-parent refusal is tested for a link above the listed file's own directory and below the root, for both ignore routes. - The fixture owner tests bound every wait and clean up in an after-hook, so a regression fails within the test and leaves no process; packaging.md and the register record round 7. Gates: lint:tools, typecheck:tools, every pnpm run ci step except the three control families (their captured sources are unchanged), with the real producer and consumer in smoke:cli-package, and the acceptance tools under CI=true GITHUB_ACTIONS=true; a mutation run of five mutants over the corrections kills all of them. Refs: ADR-0111, ADR-0051 Co-Authored-By: Claude Opus 5.5 --- docs/reference/cli/packaging.md | 2 +- .../phase-2.5.5-hardening-and-remediation.md | 1 + tools/cli-package/candidate.test.mjs | 108 ++++++---- tools/cli-package/process.mjs | 10 +- tools/cli-package/process.test.mjs | 3 +- .../cli-package/smoke/native-process.test.mjs | 187 +++++++++--------- 6 files changed, 169 insertions(+), 142 deletions(-) diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 13340ef3..1b09bfd0 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -241,7 +241,7 @@ describes the implemented mechanism; independent acceptance is recorded in the the identity `dirty` whatever their bytes), copies every file Git would ship (tracked plus untracked-but-not-ignored, minus deletions, by the repository's ignore rules and the developer's global excludes file, passed explicitly; that Git reads no home, global - or system configuration) into a fresh owner-only tree, and refuses if any of the three + or system configuration and ignores replace refs) into a fresh owner-only tree, and refuses if any of the three changed during the copy. A link, as a listed file or as any parent directory of one below the checkout, refuses the copy, so a tracked directory replaced by an ignored link cannot bring in bytes from outside the checkout. diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 485237d6..b59679a0 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -9587,6 +9587,7 @@ adversarially verified; the detailed reports stay local and Git-ignored. | 4 | `aa5c994a` | NONCLEAN (integration and critic lenses clean) | 10 (3 medium, 5 low, 2 nits) | an interrupted consumer killed the installed smoke's worker before its own removal, so the worker's directory (42 MB) stayed under the report directory and the native fixture target the installed CLI had started in its own process group ran on as an orphan: the fixture's processes now end once the smoke worker that owns their case is gone, and the consumer lifetime, given the report directory as a required `supportRoot`, abandons the bootstrap witness's children and fixtures and the smoke's worker roots itself before removing the installation. The local check's ordinary failures, its successful summary and its hand-off of the store and excludes variables, a dirty identity's path to the seal, the producer's removal of engine build state and the bounded children's process group gained killing tests. "Owner-only" is qualified where POSIX modes are not observable (Windows); the candidate copy's umask wording and the checkpoint's scope list were corrected | | 5 | `d51a2437` | NONCLEAN (plan, correctness and integration lenses clean) | 7 (1 medium, 6 low) | `prepareCandidate`'s link verification had no composed test, so deleting the call passed: a preparation whose install resolves an engine package or the root dependency directory to the original checkout now refuses after the install. A tracked directory replaced by an ignored link let the copy read bytes from outside the checkout through the linked parent, so every parent directory below the checkout must now be an ordinary directory; a `core.excludesFile` that is present but empty disables Git's global excludes and no longer falls back to the default ignore file. Settled children leaving both live-child sets, the exact `export {};` rule, each field of a facade's empty map, an interrupt the consumer outlives and the smoke-root prefix limit gained killing tests | | 6 | `3c6c5dce` | NONCLEAN (plan and correctness lenses clean) | 6 (1 medium, 4 low, 1 partial nit; 4 distinct) | nothing pinned that a leader-exit descendant watches the smoke worker rather than its own parent, so a descendant watching its exited parent would have ended on its own and left the product's group reap unproven: a test now keeps the descendant alive past its parent's exit until the owner is killed. The staging directory's ownership check gained a killing test (a different `process.getuid`); every wait for the local check's exit is bounded; and five live status statements that still said Steps 9–19 had not started now say that Step 9a.1 is under review | +| 7 | `0e59f5ff` | NONCLEAN (plan, security, correctness and integration lenses clean) | 3 (3 low) | a replace ref on HEAD let Git's status compare the working tree with a substituted commit, so the identity recorded the real HEAD as clean over other bytes: Git reading the checkout now ignores replace refs (`GIT_NO_REPLACE_OBJECTS`), with a real-repository test. The linked-parent refusal is tested for links above the listed file's own directory and below the root, and the fixture owner tests bound every wait and clean up in an after-hook, so a regression fails within the test and leaves no process | **Remaining gates.** The increment is not independently accepted. Two NEW consecutive clean complete cumulative rounds after the last material fix and a separate security sitting remain (the Acceptance diff --git a/tools/cli-package/candidate.test.mjs b/tools/cli-package/candidate.test.mjs index 17c482d2..c29ae57e 100644 --- a/tools/cli-package/candidate.test.mjs +++ b/tools/cli-package/candidate.test.mjs @@ -1,6 +1,7 @@ import assert from 'node:assert/strict'; import { chmodSync, + existsSync, mkdirSync, mkdtempSync, readFileSync, @@ -338,38 +339,61 @@ test("no configuration found inside the checkout acts as Git's global configurat assert.equal((await sourceIdentity(repository)).dirty, true); }); +const COMMIT = ['-c', 'user.name=t', '-c', 'user.email=t@example.invalid']; + for (const ignoredBy of ['.gitignore', '.git/info/exclude']) - test( - `a tracked directory replaced by a link ignored through ${ignoredBy} refuses the copy`, - { skip: !posixModes }, - async () => { - const { repository, inRepository } = await committedRepository('linked-parent'); - write(join(repository, 'a', 'b'), 'tracked\n'); - await inRepository(['add', 'a/b']); - await inRepository([ - '-c', - 'user.name=t', - '-c', - 'user.email=t@example.invalid', - 'commit', - '--quiet', - '-m', - 'two', - ]); - // The directory moves outside the checkout and comes back as a link that Git ignores. - const outside = join(fresh('outside'), 'a'); - renameSync(join(repository, 'a'), outside); - write(join(outside, 'b'), 'outside bytes\n'); - symlinkSync(outside, join(repository, 'a'), linkKind); - write(join(repository, ...ignoredBy.split('/')), 'a\n'); - const files = await candidateFiles(repository); - assert.ok(files.includes('a/b')); - assert.throws( - () => copyCandidateTree({ repository, destination: fresh('copy'), files }), - /CANDIDATE_PATH/u, - ); - }, + for (const linked of ['a', 'a/b']) + test( + `a tracked directory ${linked} replaced by a link ignored through ${ignoredBy} refuses the copy`, + { skip: !posixModes }, + async () => { + const { repository, inRepository } = await committedRepository('linked-parent'); + write(join(repository, 'a', 'b', 'c'), 'tracked\n'); + await inRepository(['add', 'a/b/c']); + await inRepository([...COMMIT, 'commit', '--quiet', '-m', 'two']); + // The directory moves outside the checkout and comes back as a link that Git ignores; a link + // above the listed file's own directory must refuse as well as its direct parent. + const outside = join(fresh('outside'), 'moved'); + renameSync(join(repository, ...linked.split('/')), outside); + write(join(outside, ...'a/b/c'.slice(linked.length + 1).split('/')), 'outside bytes\n'); + symlinkSync(outside, join(repository, ...linked.split('/')), linkKind); + write(join(repository, ...ignoredBy.split('/')), `/${linked}\n`); + const files = await candidateFiles(repository); + assert.ok(files.includes('a/b/c')); + const destination = fresh('copy'); + assert.throws( + () => copyCandidateTree({ repository, destination, files }), + /CANDIDATE_PATH/u, + ); + assert.equal(existsSync(join(destination, 'a', 'b', 'c')), false); + }, + ); + +test('a replace ref on HEAD cannot hide a substituted tree from the identity', async () => { + const { repository, inRepository } = await committedRepository('replaced'); + const reviewed = (await inRepository(['rev-parse', 'HEAD'])).stdout.trim(); + // An unrelated commit with other bytes, then a replace ref that substitutes it for HEAD and an + // index and working tree that match it, as `git reset --hard` under that ref would leave them. + write(join(repository, 'kept.ts'), 'export const substituted = true;\n'); + await inRepository(['add', 'kept.ts']); + const tree = (await inRepository(['write-tree'])).stdout.trim(); + const other = (await inRepository([...COMMIT, 'commit-tree', tree, '-m', 'other'])).stdout.trim(); + await inRepository(['replace', reviewed, other]); + await inRepository(['read-tree', '--reset', '-u', other]); + // Git honouring the replace ref sees a clean tree; the identity must not. + const honouring = Object.fromEntries( + Object.entries(isolatedGitEnvironment()).filter(([key]) => key !== 'GIT_NO_REPLACE_OBJECTS'), ); + const status = await runCommand('git', ['status', '--porcelain=v1'], { + cwd: repository, + env: honouring, + }); + assert.equal(status.stdout, ''); + const identity = await sourceIdentity(repository); + assert.equal(identity.source, reviewed); + assert.match(identity.status, /kept\.ts/u); + assert.equal(identity.dirty, true); +}); for (const flag of ['--skip-worktree', '--assume-unchanged']) test(`a tracked file marked ${flag} makes the identity dirty`, async () => { @@ -427,16 +451,22 @@ test('the candidate copy is byte-exact, keeps only the executable bit and refuse // A link as a parent directory refuses too, not only as the file itself. const elsewhere = fresh('elsewhere'); write(join(elsewhere, 'inner.ts'), 'outside\n'); + write(join(elsewhere, 'sub', 'inner.ts'), 'outside\n'); symlinkSync(elsewhere, join(repository, 'linked-directory'), linkKind); - assert.throws( - () => - copyCandidateTree({ - repository, - destination: fresh('copy'), - files: ['linked-directory/inner.ts'], - }), - /CANDIDATE_PATH/u, - ); + mkdirSync(join(repository, 'real')); + symlinkSync(elsewhere, join(repository, 'real', 'linked'), linkKind); + // The link may be the file's direct parent, a directory above it or a directory below the root. + for (const path of [ + 'linked-directory/inner.ts', + 'linked-directory/sub/inner.ts', + 'real/linked/inner.ts', + ]) { + assert.throws( + () => copyCandidateTree({ repository, destination: fresh('copy'), files: [path] }), + /CANDIDATE_PATH/u, + path, + ); + } symlinkSync(join(repository, 'package.json'), join(repository, 'linked.json')); assert.throws( () => copyCandidateTree({ repository, destination: fresh('copy'), files: ['linked.json'] }), diff --git a/tools/cli-package/process.mjs b/tools/cli-package/process.mjs index efb5771b..07915c67 100644 --- a/tools/cli-package/process.mjs +++ b/tools/cli-package/process.mjs @@ -30,10 +30,16 @@ export function isolatedEnvironment(home) { * Git with no global or system configuration, for reading the original checkout: its own `.git/config` * and ignore rules still apply and a caller passes the developer's global excludes explicitly, but no * `.gitconfig`, `git/config`, `git/ignore` or `git/attributes` in a home, configuration or system - * location is read — not even one inside the checkout itself. + * location is read — not even one inside the checkout itself. Replace refs are ignored, so a status + * compares the working tree with the commit HEAD names, never a substitute for it. */ export function isolatedGitEnvironment() { - return { ...isolatedEnvironment(devNull), GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: devNull }; + return { + ...isolatedEnvironment(devNull), + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: devNull, + GIT_NO_REPLACE_OBJECTS: '1', + }; } /** Resolve reviewed pnpm JS shims or the action's standalone executable, without cmd.exe. */ diff --git a/tools/cli-package/process.test.mjs b/tools/cli-package/process.test.mjs index 5cdba77e..a95777e7 100644 --- a/tools/cli-package/process.test.mjs +++ b/tools/cli-package/process.test.mjs @@ -24,12 +24,13 @@ import { stopLiveCommands, } from './process.mjs'; -test('Git reading the original checkout gets no home, global or system configuration', () => { +test('Git reading the original checkout gets no home, global or system configuration nor replace refs', () => { const env = isolatedGitEnvironment(); assert.deepEqual(env, { ...isolatedEnvironment(devNull), GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: devNull, + GIT_NO_REPLACE_OBJECTS: '1', }); for (const key of ['HOME', 'USERPROFILE', 'XDG_CONFIG_HOME']) assert.equal(env[key], devNull); }); diff --git a/tools/cli-package/smoke/native-process.test.mjs b/tools/cli-package/smoke/native-process.test.mjs index 7ea7c9a3..fac06229 100644 --- a/tools/cli-package/smoke/native-process.test.mjs +++ b/tools/cli-package/smoke/native-process.test.mjs @@ -326,117 +326,106 @@ function alive(pid) { } } +/** SIGKILL `pid` (a negative one names a process group) unless it is already gone. */ +function stop(pid) { + try { + process.kill(pid, 'SIGKILL'); + } catch (error) { + if (error.code !== 'ESRCH') throw error; + } +} + +/** + * Run the real fixture target in `mode` with a stand-in for the smoke worker as its owner. The target + * is detached, as the installed CLI starts it, so nothing stops its group but the fixture's own watch + * of its owner. Cleanup is registered with `t.after`, so it also runs when the test times out, and + * neither child keeps the test process alive. + */ +function ownedFixture(t, mode) { + const root = mkdtempSync(join(tmpdir(), `relavium-status-${mode}-test-`)); + const log = join(root, 'native.ndjson'); + const owner = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { + stdio: 'ignore', + }); + owner.unref(); + copyFileSync( + fileURLToPath(new URL('./native-status.cjs', import.meta.url)), + join(root, 'status'), + ); + writeFileSync( + join(root, 'native-fixture.json'), + JSON.stringify({ + mode, + log, + node: process.execPath, + descendant: fileURLToPath(new URL('./native-descendant.cjs', import.meta.url)), + owner: owner.pid, + }), + ); + const target = spawn(process.execPath, [join(root, 'status')], { + cwd: root, + detached: process.platform !== 'win32', + stdio: ['ignore', 'pipe', 'ignore'], + }); + target.unref(); + const fixture = { root, log, owner, target, stdout: '', descendant: undefined }; + target.stdout.on('data', (chunk) => { + fixture.stdout += chunk.toString(); + }); + t.after(() => { + if (process.platform !== 'win32' && target.pid !== undefined) stop(-target.pid); + else if (target.exitCode === null && target.signalCode === null) target.kill('SIGKILL'); + if (fixture.descendant !== undefined) stop(fixture.descendant); + if (owner.pid !== undefined) stop(owner.pid); + target.stdout.destroy(); + rmSync(root, { recursive: true, force: true }); + }); + return fixture; +} + +/** The descendant the fixture target recorded. */ +function recordedDescendant(fixture) { + const pid = ndjson(readFileSync(fixture.log, 'utf8')).find( + (item) => item.operation === 'native-fixture-descendant', + )?.pid; + assert.ok(Number.isSafeInteger(pid) && pid > 0); + fixture.descendant = pid; + return pid; +} + test( 'the fixture target and its descendant end once the worker that owns their case is gone', { timeout: 15000 }, - async () => { - const root = mkdtempSync(join(tmpdir(), 'relavium-status-owner-test-')); - const log = join(root, 'native.ndjson'); - // A stand-in for the smoke worker that an interrupted consumer kills. - const owner = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { - stdio: 'ignore', - }); - copyFileSync( - fileURLToPath(new URL('./native-status.cjs', import.meta.url)), - join(root, 'status'), + async (t) => { + const fixture = ownedFixture(t, 'injected-error'); + await poll( + () => fixture.stdout.includes('RELAVIUM_NATIVE_FIXTURE_READY\n'), + 5000, + 'Fixture descendant readiness missing', ); - writeFileSync( - join(root, 'native-fixture.json'), - JSON.stringify({ - mode: 'injected-error', - log, - node: process.execPath, - descendant: fileURLToPath(new URL('./native-descendant.cjs', import.meta.url)), - owner: owner.pid, - }), - ); - // Detached, as the installed CLI starts its native target, so nothing stops its group but the - // fixture's own watch of its owner. - const target = spawn(process.execPath, [join(root, 'status')], { - cwd: root, - detached: process.platform !== 'win32', - stdio: ['ignore', 'pipe', 'ignore'], - }); - let stdout = ''; - target.stdout.on('data', (chunk) => { - stdout += chunk.toString(); - }); - const pids = []; - try { - await poll( - () => stdout.includes('RELAVIUM_NATIVE_FIXTURE_READY\n'), - 5000, - 'Fixture descendant readiness missing', - ); - const descendant = ndjson(readFileSync(log, 'utf8')).find( - (item) => item.operation === 'native-fixture-descendant', - )?.pid; - assert.ok(Number.isSafeInteger(descendant) && descendant > 0); - pids.push(target.pid, descendant); - // Both stay while their owner lives. - await sleep(400); - assert.ok(pids.every(alive)); - owner.kill('SIGKILL'); - await poll( - () => !pids.some(alive), - 5000, - 'A fixture process outlived the worker that owns it', - ); - } finally { - for (const pid of pids) if (alive(pid)) process.kill(pid, 'SIGKILL'); - if (owner.exitCode === null && owner.signalCode === null) owner.kill('SIGKILL'); - rmSync(root, { recursive: true, force: true }); - } + const pids = [fixture.target.pid, recordedDescendant(fixture)]; + // Both stay while their owner lives. + await sleep(400); + assert.ok(pids.every(alive)); + fixture.owner.kill('SIGKILL'); + await poll(() => !pids.some(alive), 5000, 'A fixture process outlived the worker that owns it'); }, ); test( 'a leader-exit descendant outlives its exited parent until the worker that owns it is gone', { timeout: 15000, skip: process.platform === 'win32' }, - async () => { + async (t) => { // Leader-exit mode reparents the descendant by design; it must watch the worker, never its parent, // or it would end on its own and the product's group reap would go unproven. - const root = mkdtempSync(join(tmpdir(), 'relavium-status-leader-test-')); - const log = join(root, 'native.ndjson'); - const owner = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { - stdio: 'ignore', - }); - copyFileSync( - fileURLToPath(new URL('./native-status.cjs', import.meta.url)), - join(root, 'status'), - ); - writeFileSync( - join(root, 'native-fixture.json'), - JSON.stringify({ - mode: 'leader-exit', - log, - node: process.execPath, - descendant: fileURLToPath(new URL('./native-descendant.cjs', import.meta.url)), - owner: owner.pid, - }), - ); - const target = spawn(process.execPath, [join(root, 'status')], { - cwd: root, - detached: true, - stdio: ['ignore', 'ignore', 'ignore'], - }); - const leaderExit = new Promise((resolve) => target.once('exit', (code) => resolve(code))); - let descendant; - try { - assert.equal(await leaderExit, 0); - descendant = ndjson(readFileSync(log, 'utf8')).find( - (item) => item.operation === 'native-fixture-descendant', - )?.pid; - assert.ok(Number.isSafeInteger(descendant) && descendant > 0); - await sleep(600); - assert.equal(alive(descendant), true, 'the descendant ended with its parent, not its owner'); - owner.kill('SIGKILL'); - await poll(() => !alive(descendant), 5000, 'The descendant outlived the worker that owns it'); - } finally { - if (descendant !== undefined && alive(descendant)) process.kill(descendant, 'SIGKILL'); - if (owner.exitCode === null && owner.signalCode === null) owner.kill('SIGKILL'); - rmSync(root, { recursive: true, force: true }); - } + const fixture = ownedFixture(t, 'leader-exit'); + await poll(() => fixture.target.exitCode !== null, 5000, 'The leader did not exit on its own'); + assert.equal(fixture.target.exitCode, 0); + const descendant = recordedDescendant(fixture); + await sleep(600); + assert.equal(alive(descendant), true, 'the descendant ended with its parent, not its owner'); + fixture.owner.kill('SIGKILL'); + await poll(() => !alive(descendant), 5000, 'The descendant outlived the worker that owns it'); }, ); From 08f296cd17d51d7c1128fa7873b6b7d088fce6c0 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Thu, 22 Oct 2026 15:53:35 +0300 Subject: [PATCH 30/97] fix(repo): address the eighth Step 9a.1 review round Round 8 (at c691ef9f) reported two verified low findings and two nits, all from its tests lens; plan, security, correctness, integration and the critic were clean. - The finalizer's pre-write refusals for a non-flat runtime output and for a runtime output whose own map entry was swapped are tested. - The excludes lookup's forwarded configuration-location variables are pinned, with real-Git cases for an excludes file set only in an XDG configuration directory or a GIT_CONFIG_GLOBAL file. - Further refusals gained tests (upstream name type, metafile shape, metafile and map byte bounds, the pinned pnpm version), as did the producer's exact modes under a restrictive umask; the reference index row and the register record round 8. Gates: lint:tools, typecheck:tools, every pnpm run ci step except the three control families (their captured sources are unchanged), with the real producer and consumer in smoke:cli-package, and the acceptance tools under CI=true GITHUB_ACTIONS=true; a mutation run of thirteen mutants over the corrections kills all of them. Refs: ADR-0111, ADR-0051 Co-Authored-By: Claude Opus 5.5 --- docs/reference/cli/README.md | 2 +- .../phase-2.5.5-hardening-and-remediation.md | 1 + tools/cli-package/candidate.test.mjs | 58 ++++++++++++++++++- tools/cli-package/finalize.test.mjs | 43 ++++++++++++++ tools/cli-package/produce.test.mjs | 55 ++++++++++++++++++ 5 files changed, 155 insertions(+), 4 deletions(-) diff --git a/docs/reference/cli/README.md b/docs/reference/cli/README.md index bb56d98f..cca4004f 100644 --- a/docs/reference/cli/README.md +++ b/docs/reference/cli/README.md @@ -13,5 +13,5 @@ Part of [reference/](../README.md). | [commands.md](commands.md) | The full `relavium` command surface (run, chat, chat-resume/list/export, agent run, list, create, import, export, logs, gate, init, provider) plus interactive TUI vs CI JSON mode. | | [home.md](home.md) | The bare-invocation interactive **Home** — the TTY gate, the read-only management strip over `history.db`, the Home↔chat mode machine + keymap, bracketed paste, the min-terminal degrade, and the signal/exit-code lifecycle (2.5.B / ADR-0054). | | [chat-session.md](chat-session.md) | The `relavium chat` agent-session REPL — entry, agent/model selection, the multi-turn loop, streaming, tool availability, the `--json` session-event stream, and exit code 4. | -| [packaging.md](packaging.md) | Wave 3 finite runtime graph, installed-package consumers and Step 8 native activation accepted on `development`; Step 9 source maps, encrypted custody and release retry remain staged. | +| [packaging.md](packaging.md) | Wave 3 finite runtime graph, installed-package consumers and Step 8 native activation accepted on `development`; the Step 9a.1 candidate build and finalizer are implemented (acceptance pending); the remaining Step 9 map/custody and release gates remain staged. | | [agent-run-fixture.md](agent-run-fixture.md) | The `relavium agent run --fixture` cassette format — a `StreamChunk[][]` JSON recording of an agent run's LLM stream, for deterministic offline replay. | diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index b59679a0..3f4098ed 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -9588,6 +9588,7 @@ adversarially verified; the detailed reports stay local and Git-ignored. | 5 | `d51a2437` | NONCLEAN (plan, correctness and integration lenses clean) | 7 (1 medium, 6 low) | `prepareCandidate`'s link verification had no composed test, so deleting the call passed: a preparation whose install resolves an engine package or the root dependency directory to the original checkout now refuses after the install. A tracked directory replaced by an ignored link let the copy read bytes from outside the checkout through the linked parent, so every parent directory below the checkout must now be an ordinary directory; a `core.excludesFile` that is present but empty disables Git's global excludes and no longer falls back to the default ignore file. Settled children leaving both live-child sets, the exact `export {};` rule, each field of a facade's empty map, an interrupt the consumer outlives and the smoke-root prefix limit gained killing tests | | 6 | `3c6c5dce` | NONCLEAN (plan and correctness lenses clean) | 6 (1 medium, 4 low, 1 partial nit; 4 distinct) | nothing pinned that a leader-exit descendant watches the smoke worker rather than its own parent, so a descendant watching its exited parent would have ended on its own and left the product's group reap unproven: a test now keeps the descendant alive past its parent's exit until the owner is killed. The staging directory's ownership check gained a killing test (a different `process.getuid`); every wait for the local check's exit is bounded; and five live status statements that still said Steps 9–19 had not started now say that Step 9a.1 is under review | | 7 | `0e59f5ff` | NONCLEAN (plan, security, correctness and integration lenses clean) | 3 (3 low) | a replace ref on HEAD let Git's status compare the working tree with a substituted commit, so the identity recorded the real HEAD as clean over other bytes: Git reading the checkout now ignores replace refs (`GIT_NO_REPLACE_OBJECTS`), with a real-repository test. The linked-parent refusal is tested for links above the listed file's own directory and below the root, and the fixture owner tests bound every wait and clean up in an after-hook, so a regression fails within the test and leaves no process | +| 8 | `c691ef9f` | NONCLEAN (tests lens only; plan, security, correctness, integration and critic clean) | 4 (2 low, 2 nits) | two of the finalizer's pre-write refusals (a runtime output that is not a flat `dist/.js`, and one whose own map entry was swapped for another) and the excludes lookup's forwarded configuration-location variables had no killing test; they now do, with real-Git cases for an excludes file set only in an XDG configuration directory or a `GIT_CONFIG_GLOBAL` file. The non-material gaps were closed too: the upstream map's name type, a metafile without outputs or inputs, the metafile and map byte bounds, the pinned pnpm version, the producer's exact modes under a restrictive umask and the reference index row | **Remaining gates.** The increment is not independently accepted. Two NEW consecutive clean complete cumulative rounds after the last material fix and a separate security sitting remain (the Acceptance diff --git a/tools/cli-package/candidate.test.mjs b/tools/cli-package/candidate.test.mjs index c29ae57e..3c7cdeca 100644 --- a/tools/cli-package/candidate.test.mjs +++ b/tools/cli-package/candidate.test.mjs @@ -161,19 +161,36 @@ test("the developer's global excludes file is located without the rest of the gl error.result = { code: 1, stdout: '', stderr: '' }; throw error; }; + // Every variable Git uses to locate its configuration is forwarded, and nothing else. + const locations = { + HOME: home, + USERPROFILE: home, + XDG_CONFIG_HOME: join(home, 'xdg-absent'), + GIT_CONFIG_GLOBAL: join(home, 'global-config'), + GIT_CONFIG_SYSTEM: join(home, 'system-config'), + GIT_CONFIG_NOSYSTEM: '1', + SystemRoot: 'C:\\Windows', + WINDIR: 'C:\\Windows', + }; const environment = { PATH: '/usr/bin', - HOME: home, + ...locations, NPM_TOKEN: 'synthetic-secret', NODE_OPTIONS: '--require=/tmp/preload.cjs', GIT_DIR: '/elsewhere', }; assert.equal( - await globalExcludesFile({ repository: '/repository', environment, run: unset }), + await globalExcludesFile({ + repository: '/repository', + environment: { ...environment, XDG_CONFIG_HOME: '' }, + run: unset, + }), ignore, ); assert.deepEqual(calls[0].args, ['config', '--path', '--get', 'core.excludesFile']); - assert.deepEqual(calls[0].env, { PATH: '/usr/bin', HOME: home }); + assert.deepEqual(calls[0].env, { PATH: '/usr/bin', ...locations, XDG_CONFIG_HOME: '' }); + await globalExcludesFile({ repository: '/repository', environment, run: unset }); + assert.deepEqual(calls[1].env, { PATH: '/usr/bin', ...locations }); // Git's default prefers XDG_CONFIG_HOME, and a missing file means no global excludes. const xdg = fresh('xdg'); write(join(xdg, 'git', 'ignore'), '*.local\n'); @@ -300,6 +317,41 @@ test('a real developer configuration with an empty core.excludesFile applies no ); }); +for (const [label, place] of [ + [ + 'an XDG configuration directory', + (root, ignore) => { + write(join(root, 'xdg', 'git', 'config'), `[core]\n\texcludesFile = ${ignore}\n`); + return { XDG_CONFIG_HOME: join(root, 'xdg') }; + }, + ], + [ + 'a GIT_CONFIG_GLOBAL file', + (root, ignore) => { + write(join(root, 'dotfiles', 'gitconfig'), `[core]\n\texcludesFile = ${ignore}\n`); + return { GIT_CONFIG_GLOBAL: join(root, 'dotfiles', 'gitconfig') }; + }, + ], +]) + test(`a real core.excludesFile set only in ${label} is found`, async () => { + const root = fresh('developer'); + const ignore = join(root, 'my-ignore'); + write(ignore, 'secret.local\n'); + mkdirSync(join(root, 'empty-home')); + assert.equal( + await globalExcludesFile({ + repository: fresh('repository'), + environment: { + PATH: process.env.PATH, + HOME: join(root, 'empty-home'), + GIT_CONFIG_NOSYSTEM: '1', + ...place(root, ignore), + }, + }), + ignore, + ); + }); + /** A real repository with one committed file, `kept.ts`, created with Git's own configuration off. */ async function committedRepository(name) { const repository = fresh(name); diff --git a/tools/cli-package/finalize.test.mjs b/tools/cli-package/finalize.test.mjs index 27d6063e..fefcf480 100644 --- a/tools/cli-package/finalize.test.mjs +++ b/tools/cli-package/finalize.test.mjs @@ -255,6 +255,7 @@ for (const [label, change, code] of [ ['another file', (map) => (map.file = 'other.js'), 'UPSTREAM_MAP_SHAPE'], ['no sources', (map) => (map.sources = []), 'UPSTREAM_MAP_SHAPE'], ['malformed mappings', (map) => (map.mappings = 'AAAA!'), 'UPSTREAM_MAP_SHAPE'], + ['a non-string name', (map) => (map.names = [1]), 'UPSTREAM_MAP_SHAPE'], // Only a module whose types all erase (`export {};`, as types.js is) may map nothing. ['no mappings for executable code', (map) => (map.mappings = ''), 'UPSTREAM_MAP_UNMAPPED'], [ @@ -456,12 +457,54 @@ test('refuses an orphan map entry and validates the upstream map of a zero-byte metafile.outputs['dist/orphan.js.map'] = { imports: [], exports: [], inputs: {}, bytes: 1 }; }); refusesBeforeWriting(orphan, 'MAP_PAIRING'); + // A runtime output whose own map entry was swapped for another keeps the entry count even. + const swapped = await fixture(); + editJson(join(swapped.dist, 'metafile-esm.json'), (metafile) => { + metafile.outputs['dist/orphan.js.map'] = metafile.outputs['dist/index.js.map']; + delete metafile.outputs['dist/index.js.map']; + }); + refusesBeforeWriting(swapped, 'MAP_PAIRING'); + // A runtime output must be a flat dist/.js, even when its basename matches the inventory. + const nested = await fixture(); + editJson(join(nested.dist, 'metafile-esm.json'), (metafile) => { + const path = Object.keys(metafile.outputs).find( + (key) => key.endsWith('.js') && key !== 'dist/index.js', + ); + const moved = `dist/x/../${basename(path)}`; + metafile.outputs[moved] = metafile.outputs[path]; + metafile.outputs[`${moved}.map`] = metafile.outputs[`${path}.map`]; + delete metafile.outputs[path]; + delete metafile.outputs[`${path}.map`]; + }); + refusesBeforeWriting(nested, 'OUTPUT_INVENTORY'); // types.js contributes no byte to any output, yet its map is a consumed upstream map. const zero = await fixture(); editJson(join(zero.core, 'dist', 'types.js.map'), (map) => (map.sourceRoot = 'src')); refusesBeforeWriting(zero, 'UPSTREAM_MAP_SHAPE'); }); +test('refuses a metafile without outputs or inputs', async () => { + for (const field of ['outputs', 'inputs']) { + const candidate = await fixture(); + editJson(join(candidate.dist, 'metafile-esm.json'), (metafile) => delete metafile[field]); + refusesBeforeWriting(candidate, 'METAFILE_SHAPE'); + } +}); + +test('refuses a metafile, an upstream map or a final map over its byte bound', async () => { + // Valid JSON padded past each bound, so only the bound refuses it. + for (const [path, bound] of [ + [(candidate) => join(candidate.dist, 'metafile-esm.json'), 4 * 1024 * 1024], + [(candidate) => join(candidate.core, 'dist', 'index.js.map'), 4 * 1024 * 1024], + [(candidate) => join(candidate.dist, 'index.js.map'), 16 * 1024 * 1024], + ]) { + const candidate = await fixture(); + const file = path(candidate); + writeFileSync(file, readFileSync(file, 'utf8') + ' '.repeat(bound)); + refusesBeforeWriting(candidate, 'UNREADABLE_FILE'); + } +}); + test('requires an empty private staging directory outside the candidate', async (t) => { const candidate = await fixture(); for (const [staging, code] of [ diff --git a/tools/cli-package/produce.test.mjs b/tools/cli-package/produce.test.mjs index f22b8b23..5812af4e 100644 --- a/tools/cli-package/produce.test.mjs +++ b/tools/cli-package/produce.test.mjs @@ -161,6 +161,61 @@ test('the producer builds and seals only the owned candidate, never the original assert.deepEqual(readdirSync(destination), ['relavium-0.1.1.tgz']); }); +test('another pnpm than the pinned one refuses before anything is prepared', async () => { + const temporary = fresh('tmp'); + const { seen, ports: fakes } = ports(); + fakes.pnpm = async () => ({ stdout: '9.12.4\n', stderr: '' }); + await assert.rejects( + producePackage({ + repository: fresh('original-checkout'), + destination: join(fresh('artifact-parent'), 'artifact'), + environment: {}, + temporary, + ports: fakes, + }), + /PACKED_CLI_PNPM_VERSION/u, + ); + assert.equal(seen.prepare, undefined); + assert.deepEqual(leftovers(temporary), []); +}); + +test( + 'the private root and staging are exactly 0700 under a umask without the owner write bit', + { skip: !posixModes }, + async () => { + const seen = {}; + // The test's own directories exist before the umask applies. + const repository = fresh('original-checkout'); + const destination = join(fresh('artifact-parent'), 'artifact'); + const temporary = fresh('tmp'); + const previous = process.umask(0o277); + try { + await producePackage({ + repository, + destination, + environment: {}, + temporary, + ports: { + pnpm: async () => ({ stdout: '9.12.3\n', stderr: '' }), + excludesFile: async () => undefined, + prepare: async () => ({ source: SHA, dirty: false, files: 1 }), + emit: async ({ staging, destination, seal }) => { + seen.staging = statSync(staging).mode & 0o777; + seen.root = statSync(join(staging, '..')).mode & 0o777; + return seal(destination); + }, + noBuildCache: () => {}, + seal: () => ({ mode: 'compiled_package_test' }), + }, + }); + } finally { + process.umask(previous); + } + assert.deepEqual(seen, { staging: 0o700, root: 0o700 }); + assert.deepEqual(leftovers(temporary), []); + }, +); + test('a dirty source identity reaches the seal and the returned identity', async () => { const { seen, ports: fakes } = ports({ dirty: true }); const result = await producePackage({ From 954d6a628e803ab7438f9c09e4ceb710657834aa Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Thu, 22 Oct 2026 18:03:16 +0300 Subject: [PATCH 31/97] fix(repo): address the ninth Step 9a.1 review round Round 9 (at 08f296cd) reported three verified low findings and one nit; its plan, security, correctness and integration lenses were clean. - A failed production printed its build children's captured output, which can quote a malformed upstream map. The producer's entry point now prints only prepack's own fixed lines and one fixed code (production-failure.mjs), tested through the real entry point. - The local check's guard before the producer is tested with an interrupt during its own discovery. - The real-checkout excludes test no longer reads the host's system Git configuration; packaging.md, the README and the register record the corrections and round 9. Gates: lint:tools, typecheck:tools, every pnpm run ci step except the three control families (their captured sources are unchanged), with the real producer and consumer in smoke:cli-package, and the acceptance tools under CI=true GITHUB_ACTIONS=true; a mutation run of five mutants over the corrections kills all of them. Refs: ADR-0111, ADR-0051 Co-Authored-By: Claude Opus 5.5 --- docs/reference/cli/packaging.md | 4 +- .../phase-2.5.5-hardening-and-remediation.md | 1 + tools/cli-package/README.md | 4 +- tools/cli-package/candidate.test.mjs | 8 +-- tools/cli-package/check.test.mjs | 38 +++++++++++- tools/cli-package/pack.mjs | 21 +++++-- tools/cli-package/produce.test.mjs | 59 ++++++++++++++++++- tools/cli-package/production-failure.mjs | 35 +++++++++++ 8 files changed, 154 insertions(+), 16 deletions(-) create mode 100644 tools/cli-package/production-failure.mjs diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 1b09bfd0..8fed0ed0 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -292,7 +292,9 @@ Private maps, the metafile, the hash record and the candidate tree are removed w producer succeeds, fails or is interrupted (SIGINT, SIGTERM, SIGHUP); an interrupt also stops the build's child processes and empties the artifact directory, which was empty at the start, before the producer exits with the signal's code — also when it arrives after the build's last -child exited, during the seal or the final removal. An interrupted consumer likewise stops its +child exited, during the seal or the final removal. A failed production prints only prepack's own +fixed `✗ CLI …` lines and one fixed code, never a build child's captured output, which can quote a +malformed map. An interrupted consumer likewise stops its child processes, including the bootstrap witness's, and removes its installation, the witness's fixtures and the installed smoke's working directory; the native fixture targets the installed CLI starts in their own process groups end on their own once the smoke worker that owns their case is diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 3f4098ed..cf39bb1c 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -9589,6 +9589,7 @@ adversarially verified; the detailed reports stay local and Git-ignored. | 6 | `3c6c5dce` | NONCLEAN (plan and correctness lenses clean) | 6 (1 medium, 4 low, 1 partial nit; 4 distinct) | nothing pinned that a leader-exit descendant watches the smoke worker rather than its own parent, so a descendant watching its exited parent would have ended on its own and left the product's group reap unproven: a test now keeps the descendant alive past its parent's exit until the owner is killed. The staging directory's ownership check gained a killing test (a different `process.getuid`); every wait for the local check's exit is bounded; and five live status statements that still said Steps 9–19 had not started now say that Step 9a.1 is under review | | 7 | `0e59f5ff` | NONCLEAN (plan, security, correctness and integration lenses clean) | 3 (3 low) | a replace ref on HEAD let Git's status compare the working tree with a substituted commit, so the identity recorded the real HEAD as clean over other bytes: Git reading the checkout now ignores replace refs (`GIT_NO_REPLACE_OBJECTS`), with a real-repository test. The linked-parent refusal is tested for links above the listed file's own directory and below the root, and the fixture owner tests bound every wait and clean up in an after-hook, so a regression fails within the test and leaves no process | | 8 | `c691ef9f` | NONCLEAN (tests lens only; plan, security, correctness, integration and critic clean) | 4 (2 low, 2 nits) | two of the finalizer's pre-write refusals (a runtime output that is not a flat `dist/.js`, and one whose own map entry was swapped for another) and the excludes lookup's forwarded configuration-location variables had no killing test; they now do, with real-Git cases for an excludes file set only in an XDG configuration directory or a `GIT_CONFIG_GLOBAL` file. The non-material gaps were closed too: the upstream map's name type, a metafile without outputs or inputs, the metafile and map byte bounds, the pinned pnpm version, the producer's exact modes under a restrictive umask and the reference index row | +| 9 | `08f296cd` | NONCLEAN (plan, security, correctness and integration lenses clean) | 4 (3 low, 1 nit) | a failed production printed its build children's captured output word for word, and esbuild quotes a malformed upstream map in its warning, contrary to the rule that maps are excluded from every diagnostic: the producer's entry point now prints only prepack's own fixed `✗ CLI …` lines and one fixed code (`production-failure.mjs`), tested through the real entry point with a quoted-map marker. The local check's first stage guard gained a killing test (an interrupt during its own discovery), and the real-checkout excludes test no longer reads the host's system Git configuration | **Remaining gates.** The increment is not independently accepted. Two NEW consecutive clean complete cumulative rounds after the last material fix and a separate security sitting remain (the Acceptance diff --git a/tools/cli-package/README.md b/tools/cli-package/README.md index 1286db75..e32e8ab1 100644 --- a/tools/cli-package/README.md +++ b/tools/cli-package/README.md @@ -47,7 +47,9 @@ exited. An interrupted consumer stops its children, the bootstrap witness's incl removes its installation, the witness's fixtures and the installed smoke's working directory; the installed CLI's detached native fixture targets end once the smoke worker that owns them is gone. `pnpm smoke:cli-package` forwards an interrupt to the producer or consumer it started instead of ending before them and starts no later stage after one; a -producer killed outright can leave its owner-only root in the temporary directory. The receipt does **not** certify symbolication, a map manifest, +producer killed outright can leave its owner-only root in the temporary directory. A failed +production prints only prepack's own fixed `✗ CLI …` lines and one fixed code +(`production-failure.mjs`), never a build child's captured output, which can quote a map. The receipt does **not** certify symbolication, a map manifest, encrypted custody or a production release candidate (Steps 9a.2–9d). No private map, metafile or per-source inventory enters the public artifact. The release workflow's `release-guard.mjs` refuses every artifact until verified release custody exists: it diff --git a/tools/cli-package/candidate.test.mjs b/tools/cli-package/candidate.test.mjs index 3c7cdeca..1cfed17a 100644 --- a/tools/cli-package/candidate.test.mjs +++ b/tools/cli-package/candidate.test.mjs @@ -29,7 +29,7 @@ import { sourceIdentity, verifyCandidateLinks, } from './candidate.mjs'; -import { isolatedEnvironment, isolatedGitEnvironment, runCommand } from './process.mjs'; +import { isolatedGitEnvironment, runCommand } from './process.mjs'; const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-candidate-tree-'))); after(() => rmSync(scratch, { recursive: true, force: true })); @@ -273,9 +273,9 @@ test("the developer's global excludes file is located without the rest of the gl test('a real checkout copies nothing its developer ignores globally', async () => { const repository = fresh('real-checkout'); - const env = isolatedEnvironment(repository); + // The fixture and the lookup both run without the host's system configuration. const inRepository = (args) => - runCommand('git', args, { cwd: repository, env, timeoutMs: 30_000 }); + runCommand('git', args, { cwd: repository, env: isolatedGitEnvironment(), timeoutMs: 30_000 }); await inRepository(['init', '--quiet']); write(join(repository, 'kept.ts'), 'export {};\n'); write(join(repository, 'secret.local'), 'private\n'); @@ -294,7 +294,7 @@ test('a real checkout copies nothing its developer ignores globally', async () = write(join(home, '.config', 'git', 'ignore'), 'secret.local\n'); const excludes = await globalExcludesFile({ repository, - environment: { PATH: process.env.PATH, HOME: home }, + environment: { PATH: process.env.PATH, HOME: home, GIT_CONFIG_NOSYSTEM: '1' }, }); assert.equal(excludes, join(home, '.config', 'git', 'ignore')); assert.deepEqual(await candidateFiles(repository, runCommand, excludes), ['kept.ts']); diff --git a/tools/cli-package/check.test.mjs b/tools/cli-package/check.test.mjs index b5b940e5..02bde441 100644 --- a/tools/cli-package/check.test.mjs +++ b/tools/cli-package/check.test.mjs @@ -13,7 +13,7 @@ import { writeFileSync, } from 'node:fs'; import { devNull, tmpdir } from 'node:os'; -import { dirname, join } from 'node:path'; +import { delimiter, dirname, join } from 'node:path'; import { after, test } from 'node:test'; import { setTimeout as delay } from 'node:timers/promises'; import { fileURLToPath } from 'node:url'; @@ -114,7 +114,7 @@ mkdirSync(process.argv[2]); * explicitly, so the check runs no pnpm; so is the excludes file unless `excludes` is false, when * the check's Git lookup sees an empty home and no global or system configuration. */ -function check({ pack, consume = CONSUMER, excludes: named = true }) { +function check({ pack, consume = CONSUMER, excludes: named = true, git }) { count += 1; const root = join(scratch, `repository-${count}`); for (const path of COPIED) { @@ -134,11 +134,17 @@ function check({ pack, consume = CONSUMER, excludes: named = true }) { writeFileSync(excludes, ''); const home = join(root, 'home'); mkdirSync(home); + // An optional `git` stand-in, first on PATH, for the check's own excludes lookup. + const bin = join(root, 'bin'); + if (git !== undefined) { + mkdirSync(bin); + writeFileSync(join(bin, 'git'), git, { mode: 0o755 }); + } writeFileSync(join(root, 'tools', 'cli-package', 'pack.mjs'), pack(markers)); writeFileSync(join(root, 'tools', 'cli-package', 'consume.mjs'), consume(markers)); const child = spawn(process.execPath, [join(root, 'tools', 'cli-package', 'check.mjs')], { env: { - PATH: process.env.PATH, + PATH: git === undefined ? process.env.PATH : `${bin}${delimiter}${process.env.PATH}`, TMPDIR: temporary, RELAVIUM_PACKAGE_STORE_DIR: store, ...(named @@ -367,3 +373,29 @@ test( } }, ); + +test( + 'an interrupt during discovery stops the check before the producer', + { skip: process.platform === 'win32' }, + async () => { + // The excludes lookup's Git signals the check, ignores the interrupt forwarded back to it and + // exits 1 (no setting), so discovery itself completes normally. + const run = check({ + pack: RECORDING_PRODUCER, + consume: PUBLISHING_CONSUMER('local'), + excludes: false, + git: `#!/bin/sh\ntrap 'exit 1' INT\nkill -INT "$PPID"\nsleep 1 &\nwait\nexit 1\n`, + }); + try { + const outcome = await exitOf(run); + assert.equal(outcome.signal, null); + assert.equal(outcome.code, 130); + assert.equal(outcome.stdout, ''); + assert.equal(existsSync(join(run.markers, 'pack-environment')), false); + assert.equal(existsSync(join(run.markers, 'consume-started')), false); + assert.deepEqual(checkRoots(run.temporary), []); + } finally { + stopAll(run); + } + }, +); diff --git a/tools/cli-package/pack.mjs b/tools/cli-package/pack.mjs index 1faa39e0..13f14ea0 100644 --- a/tools/cli-package/pack.mjs +++ b/tools/cli-package/pack.mjs @@ -1,19 +1,28 @@ /** * Command-line entry point of the package producer (`produce.mjs`): one empty artifact directory in, * one tarball and its public receipt out. Never builds in the developer's workspace and never calls - * a cached build. + * a cached build. A failure prints only `productionFailure`'s fixed lines, never a child's captured + * output, which can quote a map. */ import assert from 'node:assert/strict'; import { resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { producePackage } from './produce.mjs'; +import { productionFailure } from './production-failure.mjs'; assert.equal(process.argv.length, 3, 'one empty artifact directory required'); -const { receipt, identity } = await producePackage({ - repository: fileURLToPath(new URL('../../', import.meta.url)), - destination: resolve(process.argv[2]), - environment: process.env, -}); +let produced; +try { + produced = await producePackage({ + repository: fileURLToPath(new URL('../../', import.meta.url)), + destination: resolve(process.argv[2]), + environment: process.env, + }); +} catch (error) { + for (const line of productionFailure(error)) console.error(line); + process.exit(1); +} +const { receipt, identity } = produced; console.log( JSON.stringify({ mode: receipt.mode, diff --git a/tools/cli-package/produce.test.mjs b/tools/cli-package/produce.test.mjs index 5812af4e..bb2ed032 100644 --- a/tools/cli-package/produce.test.mjs +++ b/tools/cli-package/produce.test.mjs @@ -1,5 +1,5 @@ import assert from 'node:assert/strict'; -import { spawn } from 'node:child_process'; +import { spawn, spawnSync } from 'node:child_process'; import { existsSync, mkdirSync, @@ -18,6 +18,7 @@ import { setTimeout as delay } from 'node:timers/promises'; import { assertNoBuildCache } from './candidate.mjs'; import { isolatedEnvironment } from './process.mjs'; import { producePackage } from './produce.mjs'; +import { productionFailure } from './production-failure.mjs'; const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-produce-'))); after(() => rmSync(scratch, { recursive: true, force: true })); @@ -522,3 +523,59 @@ for (const during of ['seal', 'removal']) assert.deepEqual(readdirSync(destination), []); }, ); + +/** Output a failed build child could carry: a pnpm lifecycle header and esbuild quoting a map. */ +const QUOTED_MAP = 'zzMAPMARKERzz'; +const REFUSAL = + '✗ CLI candidate finalization refused: UPSTREAM_MAP_SHAPE (packages/core/dist/index.js.map)'; +function failedChild() { + const error = new Error('PACKED_CLI_CHILD_FAILED: 1'); + error.result = { + stdout: `> relavium@0.1.1 prepack /private/candidate/apps/cli\n1 │ {"names":["${QUOTED_MAP}"],"mappings":"AAAA;!!"}\n`, + stderr: `${REFUSAL}\n✗ CLI ${QUOTED_MAP} │ {"names":[]}\n${QUOTED_MAP}\n`, + }; + return error; +} + +test('a failed production prints only prepack diagnostics and a fixed code, never child output', () => { + assert.deepEqual(productionFailure(failedChild()), [ + REFUSAL, + '✗ CLI package production failed: PACKED_CLI_CHILD_FAILED', + ]); + // A system error gives its code, never its message with a host path; anything else is unexpected. + const missing = Object.assign(new Error("ENOENT: no such file, open '/private/x'"), { + code: 'ENOENT', + }); + assert.deepEqual(productionFailure(missing), ['✗ CLI package production failed: ENOENT']); + for (const other of [new Error('a message with /private/paths'), 'thrown text', undefined]) + assert.deepEqual(productionFailure(other), ['✗ CLI package production failed: UNEXPECTED']); +}); + +test('the producer entry point prints only the fixed failure lines and exits 1', () => { + // The real entry point and failure report, with a producer stand-in that fails as a build child. + const directory = fresh('entry'); + for (const name of ['pack.mjs', 'production-failure.mjs']) + writeFileSync(join(directory, name), readFileSync(new URL(name, import.meta.url))); + writeFileSync( + join(directory, 'produce.mjs'), + `export async function producePackage() {\n` + + ` const error = new Error('PACKED_CLI_CHILD_FAILED: 1');\n` + + ` error.result = ${JSON.stringify(failedChild().result)};\n` + + ` throw error;\n}\n`, + ); + const result = spawnSync( + process.execPath, + [join(directory, 'pack.mjs'), join(directory, 'out')], + { + encoding: 'utf8', + timeout: 20_000, + }, + ); + assert.equal(result.status, 1); + assert.equal(result.stdout, ''); + assert.equal( + result.stderr, + `${REFUSAL}\n✗ CLI package production failed: PACKED_CLI_CHILD_FAILED\n`, + ); + assert.equal(result.stderr.includes(QUOTED_MAP), false); +}); diff --git a/tools/cli-package/production-failure.mjs b/tools/cli-package/production-failure.mjs new file mode 100644 index 00000000..531bfec3 --- /dev/null +++ b/tools/cli-package/production-failure.mjs @@ -0,0 +1,35 @@ +/** + * What a failed package production may print (ADR-0111). A build child's captured output can quote a + * map — esbuild excerpts a malformed upstream map in its warning — and maps are excluded from every + * diagnostic, so that output is never printed. Only prepack's own fixed `✗ CLI …` lines, which carry + * a code and at most a repository-relative path, and one fixed code for the failure itself remain. + */ + +/** A prepack diagnostic line: its fixed prefix followed by printable ASCII only. */ +const PREPACK_DIAGNOSTIC = /^✗ CLI [ -~]{1,240}$/u; +const PRINTED_DIAGNOSTICS = 8; + +/** + * The lines a failed production prints, in order: the prepack diagnostics found in the failed child's + * captured output, then `✗ CLI package production failed: `, where the code is the error's + * leading `PACKED_CLI_*` token, else its system error code, else `UNEXPECTED`. + * @param {unknown} error + * @returns {string[]} + */ +export function productionFailure(error) { + const result = + error !== null && typeof error === 'object' && 'result' in error ? error.result : undefined; + const captured = [result?.stdout, result?.stderr].filter((text) => typeof text === 'string'); + const diagnostics = captured + .flatMap((text) => text.split(/\r?\n/u)) + .map((line) => line.trim()) + .filter((line) => PREPACK_DIAGNOSTIC.test(line)) + .slice(0, PRINTED_DIAGNOSTICS); + const message = error instanceof Error ? error.message : ''; + const system = + error !== null && typeof error === 'object' && 'code' in error ? String(error.code) : ''; + const code = + /^PACKED_CLI_[A-Z0-9_]+/u.exec(message)?.[0] ?? + (/^[A-Z][A-Z0-9_]{1,63}$/u.test(system) ? system : 'UNEXPECTED'); + return [...diagnostics, `✗ CLI package production failed: ${code}`]; +} From d890abf0ee8b68691c4225c08e4a94d3a7cc5b23 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Thu, 22 Oct 2026 21:21:53 +0300 Subject: [PATCH 32/97] fix(repo): address the tenth Step 9a.1 review round Round 10 (at 954d6a62) reported three verified low findings and one nit; its plan, correctness, integration and critic lenses were clean. - With a corepack-managed pnpm, the producer's pnpm ran from the isolated build home under the shared temporary directory, where its optional module lookups could reach a node_modules another local user created (present since Step 7b; shared-/tmp hosts only, not CI). Isolated children now carry the developer's own corepack cache (COREPACK_HOME, computed as corepack computes it); a test plants such a module and shows it never runs. - The producer's interrupt cleanup attempts every step on its own; a failure leaves one fixed line and the exit code stays the signal's, and an ordinary failed removal no longer skips the other directory. - A source root's prefix-sharing sibling is tested, and an untracked nested repository refuses under its own code; packaging.md, the README and the register record round 10. Gates: lint:tools, typecheck:tools, every pnpm run ci step except the three control families (their captured sources are unchanged), with the real producer and consumer in smoke:cli-package, and the acceptance tools under CI=true GITHUB_ACTIONS=true; a mutation run of six mutants over the corrections kills all of them. Refs: ADR-0111, ADR-0051 Co-Authored-By: Claude Opus 5.5 --- docs/reference/cli/packaging.md | 9 +- .../phase-2.5.5-hardening-and-remediation.md | 1 + tools/cli-package/README.md | 4 +- tools/cli-package/candidate.mjs | 7 +- tools/cli-package/candidate.test.mjs | 9 ++ tools/cli-package/finalize.test.mjs | 15 +++ tools/cli-package/process.mjs | 28 ++++- tools/cli-package/process.test.mjs | 62 ++++++++++- tools/cli-package/produce.mjs | 54 ++++++++-- tools/cli-package/produce.test.mjs | 100 ++++++++++++++++++ 10 files changed, 272 insertions(+), 17 deletions(-) diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 8fed0ed0..6a4a76cd 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -244,7 +244,11 @@ describes the implemented mechanism; independent acceptance is recorded in the or system configuration and ignores replace refs) into a fresh owner-only tree, and refuses if any of the three changed during the copy. A link, as a listed file or as any parent directory of one below the checkout, refuses the copy, so a tracked directory replaced by an ignored link - cannot bring in bytes from outside the checkout. + cannot bring in bytes from outside the checkout; so does an untracked nested repository, + which Git lists only as its directory. Every pnpm call runs in the private home's + isolated environment, but a corepack-managed pnpm runs from the developer's own corepack + cache (`COREPACK_HOME`, as corepack computes it), never from the temporary directory, + where another local user could plant a module on its lookup path. Ignored output, caches, dependencies and private notes never enter it. The locked closure is installed there offline, frozen and with `--ignore-scripts` from the existing content-addressed store, discovered with pnpm's store-location variables @@ -294,7 +298,8 @@ the build's child processes and empties the artifact directory, which was empty before the producer exits with the signal's code — also when it arrives after the build's last child exited, during the seal or the final removal. A failed production prints only prepack's own fixed `✗ CLI …` lines and one fixed code, never a build child's captured output, which can quote a -malformed map. An interrupted consumer likewise stops its +malformed map. A step of an interrupt's cleanup that fails leaves one fixed line, never a path, +and the producer still exits with the signal's code. An interrupted consumer likewise stops its child processes, including the bootstrap witness's, and removes its installation, the witness's fixtures and the installed smoke's working directory; the native fixture targets the installed CLI starts in their own process groups end on their own once the smoke worker that owns their case is diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index cf39bb1c..b45b1190 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -9590,6 +9590,7 @@ adversarially verified; the detailed reports stay local and Git-ignored. | 7 | `0e59f5ff` | NONCLEAN (plan, security, correctness and integration lenses clean) | 3 (3 low) | a replace ref on HEAD let Git's status compare the working tree with a substituted commit, so the identity recorded the real HEAD as clean over other bytes: Git reading the checkout now ignores replace refs (`GIT_NO_REPLACE_OBJECTS`), with a real-repository test. The linked-parent refusal is tested for links above the listed file's own directory and below the root, and the fixture owner tests bound every wait and clean up in an after-hook, so a regression fails within the test and leaves no process | | 8 | `c691ef9f` | NONCLEAN (tests lens only; plan, security, correctness, integration and critic clean) | 4 (2 low, 2 nits) | two of the finalizer's pre-write refusals (a runtime output that is not a flat `dist/.js`, and one whose own map entry was swapped for another) and the excludes lookup's forwarded configuration-location variables had no killing test; they now do, with real-Git cases for an excludes file set only in an XDG configuration directory or a `GIT_CONFIG_GLOBAL` file. The non-material gaps were closed too: the upstream map's name type, a metafile without outputs or inputs, the metafile and map byte bounds, the pinned pnpm version, the producer's exact modes under a restrictive umask and the reference index row | | 9 | `08f296cd` | NONCLEAN (plan, security, correctness and integration lenses clean) | 4 (3 low, 1 nit) | a failed production printed its build children's captured output word for word, and esbuild quotes a malformed upstream map in its warning, contrary to the rule that maps are excluded from every diagnostic: the producer's entry point now prints only prepack's own fixed `✗ CLI …` lines and one fixed code (`production-failure.mjs`), tested through the real entry point with a quoted-map marker. The local check's first stage guard gained a killing test (an interrupt during its own discovery), and the real-checkout excludes test no longer reads the host's system Git configuration | +| 10 | `954d6a62` | NONCLEAN (plan, correctness, integration and critic clean) | 4 (3 low, 1 nit) | with a corepack-managed pnpm, the producer's pnpm ran from the isolated build home under the shared temporary directory, so its optional module lookups reached a `node_modules` there that another local user could create (present since Step 7b; shared-`/tmp` POSIX hosts only, never CI): isolated children now carry the developer's own corepack cache (`COREPACK_HOME`, computed as corepack computes it), and a test plants such a module and shows it never runs. A failed removal during an interrupt escaped the signal handler with a raw stack and exit 1: each cleanup step is now attempted on its own, a failure leaves one fixed line, the exit code stays the signal's, and an ordinary failed removal no longer skips the other directory. A source root's sibling sharing its prefix gained a killing test, and an untracked nested repository refuses under its own code | **Remaining gates.** The increment is not independently accepted. Two NEW consecutive clean complete cumulative rounds after the last material fix and a separate security sitting remain (the Acceptance diff --git a/tools/cli-package/README.md b/tools/cli-package/README.md index e32e8ab1..1a8c2393 100644 --- a/tools/cli-package/README.md +++ b/tools/cli-package/README.md @@ -49,7 +49,9 @@ installed CLI's detached native fixture targets end once the smoke worker that o consumer it started instead of ending before them and starts no later stage after one; a producer killed outright can leave its owner-only root in the temporary directory. A failed production prints only prepack's own fixed `✗ CLI …` lines and one fixed code -(`production-failure.mjs`), never a build child's captured output, which can quote a map. The receipt does **not** certify symbolication, a map manifest, +(`production-failure.mjs`), never a build child's captured output, which can quote a map. A +corepack-managed pnpm always runs from the developer's own corepack cache, never from the shared +temporary directory that holds the isolated homes (`corepackHome` in `process.mjs`). The receipt does **not** certify symbolication, a map manifest, encrypted custody or a production release candidate (Steps 9a.2–9d). No private map, metafile or per-source inventory enters the public artifact. The release workflow's `release-guard.mjs` refuses every artifact until verified release custody exists: it diff --git a/tools/cli-package/candidate.mjs b/tools/cli-package/candidate.mjs index 045c3108..0ea558c0 100644 --- a/tools/cli-package/candidate.mjs +++ b/tools/cli-package/candidate.mjs @@ -164,7 +164,7 @@ export async function sourceIdentity(repository, run = runCommand, excludes = un * Every file Git would ship from the working tree: tracked plus untracked-but-not-ignored, minus * deletions, by the repository's ignore rules and the developer's global excludes * (`globalExcludesFile`). Ignored output, caches, dependencies and private notes never enter a - * candidate. + * candidate; an untracked nested repository, which Git lists only as its directory, refuses. * @param {string} repository @param {typeof runCommand} [run] @param {string} [excludes] * @param {{files: number}} [limits] the bounds (`CANDIDATE_LIMITS`; smaller ones in tests) */ @@ -191,6 +191,11 @@ export async function candidateFiles( ) ).stdout, ); + // Git lists an untracked nested repository as its directory (`dir/`), never its files. + assert.ok( + listed.every((path) => !path.endsWith('/')), + 'PACKED_CLI_CANDIDATE_NESTED_REPOSITORY', + ); const deleted = new Set( gitPaths((await run('git', ['ls-files', '-z', '--deleted'], options)).stdout), ); diff --git a/tools/cli-package/candidate.test.mjs b/tools/cli-package/candidate.test.mjs index 1cfed17a..9e746ade 100644 --- a/tools/cli-package/candidate.test.mjs +++ b/tools/cli-package/candidate.test.mjs @@ -148,6 +148,15 @@ test('candidate files are tracked plus unignored untracked files, minus deletion 'ls-files -z --deleted': '', }); assert.deepEqual(await candidateFiles('/repository', excluded.run, '/ignore'), ['a.ts']); + // An untracked nested repository is listed only as its directory and refuses by name. + const nested = git({ + 'ls-files -z --cached --others --exclude-standard': 'a.ts\0vendor/clone/\0', + 'ls-files -z --deleted': '', + }); + await assert.rejects( + candidateFiles('/repository', nested.run), + /PACKED_CLI_CANDIDATE_NESTED_REPOSITORY/u, + ); }); test("the developer's global excludes file is located without the rest of the global configuration", async () => { diff --git a/tools/cli-package/finalize.test.mjs b/tools/cli-package/finalize.test.mjs index fefcf480..1364e45e 100644 --- a/tools/cli-package/finalize.test.mjs +++ b/tools/cli-package/finalize.test.mjs @@ -483,6 +483,21 @@ test('refuses an orphan map entry and validates the upstream map of a zero-byte refusesBeforeWriting(zero, 'UPSTREAM_MAP_SHAPE'); }); +test('a source root is a whole directory: a sibling that only shares its prefix is not original', async () => { + // An upstream map naming packages/core/srcxx and a final map naming apps/cli/srcxx, both existing. + const upstream = await fixture(); + write(join(upstream.core, 'srcxx', 'index.ts'), 'export const x = 1;\n'); + editJson( + join(upstream.core, 'dist', 'index.js.map'), + (map) => (map.sources = ['../srcxx/index.ts']), + ); + refusesBeforeWriting(upstream, 'MAP_SOURCE_NOT_ORIGINAL'); + const final = await fixture(); + write(join(final.cli, 'srcxx', 'x.ts'), 'export const x = 1;\n'); + editJson(join(final.dist, 'index.js.map'), (map) => (map.sources = ['../srcxx/x.ts'])); + refusesBeforeWriting(final, 'MAP_SOURCE_NOT_ORIGINAL'); +}); + test('refuses a metafile without outputs or inputs', async () => { for (const field of ['outputs', 'inputs']) { const candidate = await fixture(); diff --git a/tools/cli-package/process.mjs b/tools/cli-package/process.mjs index 07915c67..531a2c07 100644 --- a/tools/cli-package/process.mjs +++ b/tools/cli-package/process.mjs @@ -3,10 +3,32 @@ import assert from 'node:assert/strict'; import { spawn, spawnSync } from 'node:child_process'; import { Buffer } from 'node:buffer'; import { existsSync, lstatSync, readFileSync, realpathSync, statSync } from 'node:fs'; -import { constants, devNull } from 'node:os'; -import { delimiter, extname, isAbsolute, join } from 'node:path'; +import { constants, devNull, homedir } from 'node:os'; +import { delimiter, extname, isAbsolute, join, resolve } from 'node:path'; import { clearTimeout, setImmediate, setTimeout } from 'node:timers'; +/** + * Corepack's own cache, computed as corepack computes it (`COREPACK_HOME`, else `XDG_CACHE_HOME`, else + * `LOCALAPPDATA`, else the home directory's `.cache` or `AppData/Local`, plus `node/corepack`), from + * this process's environment, which is the developer's own or carries a `COREPACK_HOME` a caller + * computed from it. An isolated child passes it on, so a corepack-managed pnpm runs from that cache — + * never from an isolated home in the shared temporary directory, where its optional module lookups + * would reach a `node_modules` that another local user can create. + * @param {Record} [environment] + */ +export function corepackHome(environment = process.env) { + return resolve( + environment.COREPACK_HOME ?? + join( + environment.XDG_CACHE_HOME ?? + environment.LOCALAPPDATA ?? + join(homedir(), process.platform === 'win32' ? join('AppData', 'Local') : '.cache'), + 'node', + 'corepack', + ), + ); +} + export function isolatedEnvironment(home) { const env = { HOME: home, @@ -20,6 +42,8 @@ export function isolatedEnvironment(home) { NO_COLOR: '1', TERM: 'dumb', COREPACK_ENABLE_DOWNLOAD_PROMPT: '0', + // A location only, never configuration: see `corepackHome`. + COREPACK_HOME: corepackHome(), }; for (const key of ['PATH', 'SystemRoot', 'WINDIR', 'LANG']) if (process.env[key] !== undefined) env[key] = process.env[key]; diff --git a/tools/cli-package/process.test.mjs b/tools/cli-package/process.test.mjs index a95777e7..28d43d89 100644 --- a/tools/cli-package/process.test.mjs +++ b/tools/cli-package/process.test.mjs @@ -11,10 +11,11 @@ import { rmSync, realpathSync, } from 'node:fs'; -import { devNull, tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { devNull, homedir, tmpdir } from 'node:os'; +import { join, resolve, sep } from 'node:path'; import { fileURLToPath } from 'node:url'; import { + corepackHome, exerciseInstalledCommand, isolatedEnvironment, isolatedGitEnvironment, @@ -306,3 +307,60 @@ test( assert.equal(kill.mock.callCount(), 0); }, ); + +test('the corepack cache is computed as corepack computes it, and isolated children keep it', () => { + assert.equal( + corepackHome({ COREPACK_HOME: '/explicit/corepack' }), + resolve('/explicit/corepack'), + ); + assert.equal( + corepackHome({ XDG_CACHE_HOME: '/xdg', LOCALAPPDATA: '/local' }), + resolve('/xdg', 'node', 'corepack'), + ); + assert.equal(corepackHome({ LOCALAPPDATA: '/local' }), resolve('/local', 'node', 'corepack')); + assert.equal( + corepackHome({}), + resolve( + homedir(), + process.platform === 'win32' ? join('AppData', 'Local') : '.cache', + 'node', + 'corepack', + ), + ); + // An isolated child carries this process's corepack cache, never one inside its own home. + const home = join(tmpdir(), 'synthetic-installer-home'); + const env = isolatedEnvironment(home); + assert.equal(env.COREPACK_HOME, corepackHome()); + assert.equal(env.COREPACK_HOME.startsWith(`${home}${sep}`), false); +}); + +test( + 'a corepack-managed pnpm never runs from the temporary directory an isolated home sits in', + { skip: process.platform === 'win32' }, + async () => { + // A module another local user could plant in the shared temporary directory, on the lookup + // path of code that runs from a home inside it. + const temporary = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-corepack-temporary-'))); + try { + const marker = join(temporary, 'planted-module-ran'); + for (const name of ['v8-compile-cache', 'supports-color']) { + mkdirSync(join(temporary, 'node_modules', name), { recursive: true }); + writeFileSync( + join(temporary, 'node_modules', name, 'index.js'), + `require('node:fs').writeFileSync(${JSON.stringify(marker)}, ${JSON.stringify(name)});`, + ); + } + const home = realpathSync(mkdtempSync(join(temporary, 'relavium-package-build-'))); + writeFileSync(join(home, 'package.json'), '{"packageManager":"pnpm@9.12.3"}\n'); + const version = await runPnpm(['--version'], { + cwd: home, + env: isolatedEnvironment(home), + timeoutMs: 120_000, + }); + assert.equal(version.stdout.trim(), '9.12.3'); + assert.equal(existsSync(marker), false, 'a module planted in the temporary directory ran'); + } finally { + rmSync(temporary, { recursive: true, force: true }); + } + }, +); diff --git a/tools/cli-package/produce.mjs b/tools/cli-package/produce.mjs index c4012c55..f47ec90a 100644 --- a/tools/cli-package/produce.mjs +++ b/tools/cli-package/produce.mjs @@ -9,7 +9,15 @@ * are tooling seams for tests, never shipped product hooks. */ import assert from 'node:assert/strict'; -import { chmodSync, mkdirSync, mkdtempSync, readdirSync, realpathSync, rmSync } from 'node:fs'; +import { + chmodSync, + mkdirSync, + mkdtempSync, + readdirSync, + realpathSync, + rmSync, + writeSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { assertNoBuildCache, globalExcludesFile, prepareCandidate } from './candidate.mjs'; @@ -22,11 +30,12 @@ import { emitFreshPackage } from './pipeline.mjs'; * Produce one package-test artifact from the original checkout `repository` into the empty * `destination`. The candidate tree and its private staging are siblings in one temporary root, * owner-only where POSIX modes are observable, that shares no path with the artifact directory or the - * checkout. That root and the build home - * are removed whether production succeeds or fails. An interrupt (SIGINT, SIGTERM, SIGHUP) stops the + * checkout. That root and the build home are removed whether production succeeds or fails, each + * attempted even when the other's removal fails. An interrupt (SIGINT, SIGTERM, SIGHUP) stops the * build's child trees, removes both directories and empties the artifact directory, which was empty * at the start, before the process exits with the signal's code — also when it arrives after the last - * child exited, during the seal or during the final removal (`exitOnInterrupt`). + * child exited, during the seal or during the final removal (`exitOnInterrupt`). A step of that + * cleanup that fails leaves one fixed line, never a path, and the exit code stays the signal's. * @param {{repository: string, destination: string, environment: Record, * temporary?: string, ports?: {prepare?: typeof prepareCandidate, emit?: typeof emitFreshPackage, * seal?: typeof sealPackage, noBuildCache?: typeof assertNoBuildCache, @@ -54,15 +63,42 @@ export async function producePackage({ assert.deepEqual(readdirSync(destination), [], 'PACKED_CLI_FRESH_DESTINATION_REQUIRED'); const home = realpathSync.native(mkdtempSync(join(temporary, 'relavium-package-build-'))); let root; + // Every removal is attempted even when an earlier one fails; the first failure is rethrown. const discard = () => { - if (root !== undefined) remove(root); - remove(home); + let failure; + for (const path of [root, home]) { + if (path === undefined) continue; + try { + remove(path); + } catch (error) { + failure ??= error; + } + } + if (failure !== undefined) throw failure; }; + // An interrupt's cleanup never throws: a failed step leaves one fixed line, without a path, and + // the producer still exits with the signal's code. const interrupted = () => { - discard(); - for (const name of readdirSync(destination)) { - rmSync(join(destination, name), { recursive: true, force: true, maxRetries: 3 }); + let failed = false; + const attempt = (step) => { + try { + step(); + } catch { + failed = true; + } + }; + attempt(discard); + let names = []; + attempt(() => { + names = readdirSync(destination); + }); + for (const name of names) { + attempt(() => + rmSync(join(destination, name), { recursive: true, force: true, maxRetries: 3 }), + ); } + if (failed) + attempt(() => writeSync(2, '✗ CLI package production interrupted: CLEANUP_FAILED\n')); }; return exitOnInterrupt(interrupted, async () => { try { diff --git a/tools/cli-package/produce.test.mjs b/tools/cli-package/produce.test.mjs index bb2ed032..84a077dc 100644 --- a/tools/cli-package/produce.test.mjs +++ b/tools/cli-package/produce.test.mjs @@ -449,6 +449,106 @@ for (const [signal, code] of [ ); } +test('a failed removal still removes the other private directory and rethrows', async () => { + const temporary = fresh('tmp'); + const { ports: fakes } = ports(); + const refusal = Object.assign(new Error('EACCES: synthetic refusal'), { code: 'EACCES' }); + fakes.remove = (path) => { + if (path.includes('relavium-package-candidate-')) throw refusal; + rmSync(path, { recursive: true, force: true }); + }; + const before = listeners(); + await assert.rejects( + producePackage({ + repository: fresh('original-checkout'), + destination: join(fresh('artifact-parent'), 'artifact'), + environment: {}, + temporary, + ports: fakes, + }), + (error) => error === refusal, + ); + // The build home is gone although the private root's removal failed first. + assert.deepEqual( + leftovers(temporary).map((name) => name.replace(/-[^-]+$/u, '')), + ['relavium-package-candidate'], + ); + assert.deepEqual(listeners(), before); +}); + +test( + 'a failed removal during an interrupt leaves one fixed line and still exits with the signal', + { skip: !posixModes }, + async () => { + const temporary = fresh('tmp'); + mkdirSync(join(temporary, 'checkout')); + const destination = join(fresh('artifact-parent'), 'artifact'); + const pidFile = join(fresh('pid'), 'build.pid'); + const url = (name) => JSON.stringify(new URL(name, import.meta.url).href); + const script = ` +import { rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { producePackage } from ${url('./produce.mjs')}; +import { isolatedEnvironment, runCommand } from ${url('./process.mjs')}; +await producePackage({ + repository: ${JSON.stringify(join(temporary, 'checkout'))}, + destination: ${JSON.stringify(destination)}, + environment: {}, + temporary: ${JSON.stringify(temporary)}, + ports: { + pnpm: async () => ({ stdout: '9.12.3\\n', stderr: '' }), + excludesFile: async () => undefined, + prepare: async () => ({ source: '${SHA}', dirty: false, files: 1 }), + emit: async ({ destination }) => { + writeFileSync(join(destination, 'relavium-0.1.1.tgz'), 'unsealed'); + await runCommand(process.execPath, ['-e', ${JSON.stringify( + `require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, String(process.pid)); setInterval(() => {}, 1000);`, + )}], { env: isolatedEnvironment(${JSON.stringify(temporary)}), timeoutMs: 60_000 }); + }, + remove: (path) => { + if (path.includes('relavium-package-candidate-')) + throw Object.assign(new Error('EACCES: permission denied, unlink ' + JSON.stringify(path)), { code: 'EACCES' }); + rmSync(path, { recursive: true, force: true }); + }, + }, +}); +`; + const child = spawn(process.execPath, ['--input-type=module', '-e', script], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stderr = ''; + child.stderr.on('data', (chunk) => (stderr += chunk)); + let build; + try { + for ( + let waited = 0; + !existsSync(pidFile) || readFileSync(pidFile, 'utf8') === ''; + waited += 25 + ) { + assert.ok(waited < 20_000, 'the fake build never started'); + await delay(25); + } + build = Number(readFileSync(pidFile, 'utf8')); + const exited = new Promise((resolve) => + child.once('exit', (code, signal) => resolve([code, signal])), + ); + child.kill('SIGINT'); + assert.deepEqual(await exited, [130, null]); + assert.equal(stderr, '✗ CLI package production interrupted: CLEANUP_FAILED\n'); + assert.equal(alive(build), false); + // The other steps still ran: the build home is gone and the artifact directory is empty. + assert.deepEqual( + leftovers(temporary).map((name) => name.replace(/-[^-]+$/u, '')), + ['relavium-package-candidate'], + ); + assert.deepEqual(readdirSync(destination), []); + } finally { + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); + if (build !== undefined && alive(build)) process.kill(build, 'SIGKILL'); + } + }, +); + /** * A producer whose interrupt arrives after its build's last child exited: during the seal or during * the private root's removal, both synchronous. The fake emit's last await completes from an I/O From 786239640f375549cadfaeed1d8becd0c26316e3 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Fri, 23 Oct 2026 21:04:43 +0300 Subject: [PATCH 33/97] fix(repo): address the eleventh Step 9a.1 review round Round 11 (at d890abf0) reported four verified findings, one medium; its plan, security, correctness and integration lenses were clean. - The composition test pins that the producer looks up the developer's global excludes and store in the caller's own environment. - A local-check stage past its deadline is interrupted first (runCommand's graceMs) and killed only after a grace period, so the producer and consumer stop their own detached build children before the check root goes; tested with a late-writing build child. - The check's wait for an interrupt caught during its own root removal is tested, dependencyStore's docblock says where corepack runs, and packaging.md, the README and the register record round 11. Gates: lint:tools, typecheck:tools, every pnpm run ci step except the three control families (their captured sources are unchanged), with the real producer and consumer in smoke:cli-package, and the acceptance tools under CI=true GITHUB_ACTIONS=true; a mutation run of six mutants over the corrections kills all of them. Refs: ADR-0111, ADR-0051 Co-Authored-By: Claude Opus 5.5 --- docs/reference/cli/packaging.md | 3 +- .../phase-2.5.5-hardening-and-remediation.md | 1 + tools/cli-package/README.md | 3 +- tools/cli-package/candidate.mjs | 5 +- tools/cli-package/check.mjs | 12 +- tools/cli-package/check.test.mjs | 111 +++++++++++++++++- tools/cli-package/process.mjs | 24 +++- tools/cli-package/process.test.mjs | 62 ++++++++++ tools/cli-package/produce.test.mjs | 10 +- 9 files changed, 219 insertions(+), 12 deletions(-) diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 6a4a76cd..8a3964df 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -305,7 +305,8 @@ fixtures and the installed smoke's working directory; the native fixture targets starts in their own process groups end on their own once the smoke worker that owns their case is gone. The local `pnpm smoke:cli-package` forwards an interrupt to the producer or consumer it started rather than ending first, starts no later stage once one arrived, and exits with the signal's -code after removing its root. A producer killed outright, a crash or a lost host can leave the +code after removing its root; a stage past its deadline is interrupted (SIGTERM) first and killed +only after a grace period, so it stops its own build children before the root goes. A producer killed outright, a crash or a lost host can leave the owner-only root in the temporary directory, and no secure erasure is claimed. "Owner-only" in this section means mode 0700 for directories and 0600 for staged files, set — and for the staging directory verified together with this process's uid — where POSIX modes are observable. On Windows diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index b45b1190..732341e0 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -9591,6 +9591,7 @@ adversarially verified; the detailed reports stay local and Git-ignored. | 8 | `c691ef9f` | NONCLEAN (tests lens only; plan, security, correctness, integration and critic clean) | 4 (2 low, 2 nits) | two of the finalizer's pre-write refusals (a runtime output that is not a flat `dist/.js`, and one whose own map entry was swapped for another) and the excludes lookup's forwarded configuration-location variables had no killing test; they now do, with real-Git cases for an excludes file set only in an XDG configuration directory or a `GIT_CONFIG_GLOBAL` file. The non-material gaps were closed too: the upstream map's name type, a metafile without outputs or inputs, the metafile and map byte bounds, the pinned pnpm version, the producer's exact modes under a restrictive umask and the reference index row | | 9 | `08f296cd` | NONCLEAN (plan, security, correctness and integration lenses clean) | 4 (3 low, 1 nit) | a failed production printed its build children's captured output word for word, and esbuild quotes a malformed upstream map in its warning, contrary to the rule that maps are excluded from every diagnostic: the producer's entry point now prints only prepack's own fixed `✗ CLI …` lines and one fixed code (`production-failure.mjs`), tested through the real entry point with a quoted-map marker. The local check's first stage guard gained a killing test (an interrupt during its own discovery), and the real-checkout excludes test no longer reads the host's system Git configuration | | 10 | `954d6a62` | NONCLEAN (plan, correctness, integration and critic clean) | 4 (3 low, 1 nit) | with a corepack-managed pnpm, the producer's pnpm ran from the isolated build home under the shared temporary directory, so its optional module lookups reached a `node_modules` there that another local user could create (present since Step 7b; shared-`/tmp` POSIX hosts only, never CI): isolated children now carry the developer's own corepack cache (`COREPACK_HOME`, computed as corepack computes it), and a test plants such a module and shows it never runs. A failed removal during an interrupt escaped the signal handler with a raw stack and exit 1: each cleanup step is now attempted on its own, a failure leaves one fixed line, the exit code stays the signal's, and an ordinary failed removal no longer skips the other directory. A source root's sibling sharing its prefix gained a killing test, and an untracked nested repository refuses under its own code | +| 11 | `d890abf0` | NONCLEAN (plan, security, correctness and integration lenses clean) | 4 (1 medium, 3 low) | nothing pinned that the producer looks up the developer's global excludes in the caller's own environment rather than the isolated build environment, which would have dropped them silently: the composition test now requires the very environment object. The local check's deadline killed only a stage's own process group, so a producer's detached build child could recreate the removed root with a map: a stage past its deadline is now interrupted first (`runCommand`'s `graceMs`) and killed only after a grace period, tested with a late-writing build child. The check's wait for an interrupt caught during its own root removal gained a killing test, and `dependencyStore`'s docblock says where corepack runs | **Remaining gates.** The increment is not independently accepted. Two NEW consecutive clean complete cumulative rounds after the last material fix and a separate security sitting remain (the Acceptance diff --git a/tools/cli-package/README.md b/tools/cli-package/README.md index 1a8c2393..e91ef897 100644 --- a/tools/cli-package/README.md +++ b/tools/cli-package/README.md @@ -46,7 +46,8 @@ stops the build first and empties the artifact directory, even after the last bu exited. An interrupted consumer stops its children, the bootstrap witness's included, and removes its installation, the witness's fixtures and the installed smoke's working directory; the installed CLI's detached native fixture targets end once the smoke worker that owns them is gone. `pnpm smoke:cli-package` forwards an interrupt to the producer or -consumer it started instead of ending before them and starts no later stage after one; a +consumer it started instead of ending before them and starts no later stage after one, and it +interrupts a stage past its deadline before it kills it, so the stage stops its own children; a producer killed outright can leave its owner-only root in the temporary directory. A failed production prints only prepack's own fixed `✗ CLI …` lines and one fixed code (`production-failure.mjs`), never a build child's captured output, which can quote a map. A diff --git a/tools/cli-package/candidate.mjs b/tools/cli-package/candidate.mjs index 0ea558c0..b7821086 100644 --- a/tools/cli-package/candidate.mjs +++ b/tools/cli-package/candidate.mjs @@ -262,8 +262,9 @@ export const PACKAGE_STORE_ENV = 'RELAVIUM_PACKAGE_STORE_DIR'; * The developer's existing content-addressed dependency store: the explicit * `RELAVIUM_PACKAGE_STORE_DIR` when a caller already isolated this process's environment, * otherwise discovered by the pinned pnpm with only the store-location variables of `environment`. - * Everything else — temporary files, caches, corepack state — goes to the private `home`, never - * into the original checkout. + * Temporary files and pnpm's caches go to the private `home`, never into the original checkout; a + * corepack-managed pnpm runs from the developer's own corepack cache (`COREPACK_HOME`, see + * `corepackHome` in process.mjs), never from the shared temporary directory. * @param {{repository: string, environment: Record, home: string, * pnpm?: typeof runPnpm}} options */ diff --git a/tools/cli-package/check.mjs b/tools/cli-package/check.mjs index 10f72ee0..208b3a3a 100644 --- a/tools/cli-package/check.mjs +++ b/tools/cli-package/check.mjs @@ -8,6 +8,8 @@ import { dependencyStore, globalExcludesFile, producerEnvironment } from './cand import { forwardInterrupts, isolatedEnvironment, runCommand } from './process.mjs'; assert.equal(process.argv.length, 2, 'no arguments required'); +/** Shortens each stage's deadline (milliseconds, at most the default 600000). */ +const DEADLINE_ENV = 'RELAVIUM_LOCAL_CHECK_DEADLINE_MS'; const directory = fileURLToPath(new URL('.', import.meta.url)); const root = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-local-package-check-'))); // The producer and the consumer run detached, and each stops its own child trees and removes its @@ -25,7 +27,15 @@ try { const excludes = await globalExcludesFile({ repository, environment: process.env }); const artifact = join(root, 'artifact'), reports = join(root, 'reports'); - const options = { env: isolatedEnvironment(root), timeoutMs: 600_000 }; + // A stage past its deadline is interrupted first (SIGTERM), so it stops its own detached build + // children and removes its private directories; only a stage that ignores that is killed. The + // deadline can only be shortened, for the tests. + const deadline = Number(process.env[DEADLINE_ENV] ?? 600_000); + assert.ok( + Number.isSafeInteger(deadline) && deadline > 0 && deadline <= 600_000, + 'PACKED_CLI_LOCAL_CHECK_DEADLINE', + ); + const options = { env: isolatedEnvironment(root), timeoutMs: deadline, graceMs: 30_000 }; proceed(); await runCommand(process.execPath, [join(directory, 'pack.mjs'), artifact], { ...options, diff --git a/tools/cli-package/check.test.mjs b/tools/cli-package/check.test.mjs index 02bde441..636d1d5f 100644 --- a/tools/cli-package/check.test.mjs +++ b/tools/cli-package/check.test.mjs @@ -114,7 +114,7 @@ mkdirSync(process.argv[2]); * explicitly, so the check runs no pnpm; so is the excludes file unless `excludes` is false, when * the check's Git lookup sees an empty home and no global or system configuration. */ -function check({ pack, consume = CONSUMER, excludes: named = true, git }) { +function check({ pack, consume = CONSUMER, excludes: named = true, git, env: extra = {} }) { count += 1; const root = join(scratch, `repository-${count}`); for (const path of COPIED) { @@ -150,6 +150,7 @@ function check({ pack, consume = CONSUMER, excludes: named = true, git }) { ...(named ? { RELAVIUM_GIT_EXCLUDES_FILE: excludes } : { HOME: home, GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: devNull }), + ...extra, }, stdio: ['ignore', 'pipe', 'pipe'], }); @@ -193,7 +194,13 @@ function alive(pid) { */ function stopAll(run) { if (run.child.exitCode === null && run.child.signalCode === null) run.child.kill('SIGKILL'); - for (const name of ['pack.pid', 'consume-started', 'writer.pid']) { + for (const name of [ + 'pack.pid', + 'consume-started', + 'writer.pid', + 'grandchild.pid', + 'helper.pid', + ]) { const file = join(run.markers, name); const pid = existsSync(file) ? Number(readFileSync(file, 'utf8')) : NaN; if (Number.isSafeInteger(pid) && pid > 0 && alive(pid)) process.kill(pid, 'SIGKILL'); @@ -399,3 +406,103 @@ test( } }, ); + +test( + 'a stage past its deadline is interrupted first, so its own children never outlive the check', + { skip: process.platform === 'win32' }, + async () => { + // A producer that handles the interrupt as the real one does, around a detached build child that + // would write a map into the check root after the check removed it. + const run = check({ + env: { RELAVIUM_LOCAL_CHECK_DEADLINE_MS: '1500' }, + pack: (markers) => ` +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { writeFileSync } from 'node:fs'; +import { exitOnInterrupt, isolatedEnvironment, runCommand } from './process.mjs'; +const late = join(tmpdir(), 'late'); +await exitOnInterrupt( + () => writeFileSync(${JSON.stringify(join(markers, 'pack-interrupted'))}, 'cleaned'), + () => + runCommand(process.execPath, ['-e', ${JSON.stringify( + `const fs = require('node:fs');` + + `fs.writeFileSync(${JSON.stringify(join(markers, 'grandchild.pid'))}, String(process.pid));`, + )} + 'setTimeout(() => { require("node:fs").mkdirSync(' + JSON.stringify(late) + ', { recursive: true }); require("node:fs").writeFileSync(' + JSON.stringify(join(late, 'index.js.map')) + ', "map"); }, 3000); setInterval(() => {}, 1000);'], { + env: isolatedEnvironment(tmpdir()), + timeoutMs: 60_000, + }), +); +`, + }); + try { + const grandchild = Number(await waitForFile(join(run.markers, 'grandchild.pid'))); + const [checkRoot] = checkRoots(run.temporary); + assert.ok(checkRoot !== undefined); + const outcome = await exitOf(run); + assert.equal(outcome.signal, null); + assert.equal(outcome.code, 1); + assert.equal(outcome.stdout, ''); + assert.equal(readFileSync(join(run.markers, 'pack-interrupted'), 'utf8'), 'cleaned'); + assert.equal(alive(grandchild), false); + // Past the moment the build child would have written: nothing recreated the check root. + await delay(3500); + assert.equal(existsSync(join(run.temporary, checkRoot)), false); + } finally { + stopAll(run); + } + }, +); + +test('only a shorter, positive stage deadline is admitted', async () => { + for (const deadline of ['0', '-1', '600001', 'soon', '']) { + const run = check({ + env: { RELAVIUM_LOCAL_CHECK_DEADLINE_MS: deadline }, + pack: RECORDING_PRODUCER, + consume: PUBLISHING_CONSUMER('local'), + }); + try { + const outcome = await exitOf(run); + assert.equal(outcome.code, 1, deadline); + assert.equal(existsSync(join(run.markers, 'pack-environment')), false, deadline); + assert.deepEqual(checkRoots(run.temporary), []); + } finally { + stopAll(run); + } + } +}); + +test( + 'an interrupt during the root removal of the check still ends it with the signal', + { skip: process.platform === 'win32' }, + async () => { + // The consumer publishes, leaves a large tree in the check root and starts a helper that signals + // the check as soon as that tree starts to shrink, that is during the check's own removal. + const removal = (markers) => + PUBLISHING_CONSUMER('local')(markers) + + ` +const bulk = join(reports, 'bulk'); +mkdirSync(bulk); +for (let i = 0; i < 20000; i += 1) writeFileSync(join(bulk, String(i)), ''); +const { spawn } = await import('node:child_process'); +const helper = spawn(process.execPath, ['-e', ${JSON.stringify( + `const fs = require('node:fs');` + + `const [bulk, check, marker] = process.argv.slice(1);` + + `const poll = () => { let count; try { count = fs.readdirSync(bulk).length; } catch { process.exit(0); }` + + ` if (count < 20000) { process.kill(Number(check), 'SIGINT'); fs.writeFileSync(marker, 'sent'); process.exit(0); }` + + ` setTimeout(poll, 1); }; poll();`, + )}, bulk, String(process.ppid), ${JSON.stringify(join(markers, 'removal-signalled'))}], { detached: true, stdio: 'ignore' }); +writeFileSync(${JSON.stringify(join(markers, 'helper.pid'))}, String(helper.pid)); +helper.unref(); +`; + const run = check({ pack: RECORDING_PRODUCER, consume: removal }); + try { + const outcome = await exitOf(run); + assert.equal(readFileSync(join(run.markers, 'removal-signalled'), 'utf8'), 'sent'); + assert.equal(outcome.signal, null); + assert.equal(outcome.code, 130); + assert.deepEqual(checkRoots(run.temporary), []); + } finally { + stopAll(run); + } + }, +); diff --git a/tools/cli-package/process.mjs b/tools/cli-package/process.mjs index 531a2c07..66af623d 100644 --- a/tools/cli-package/process.mjs +++ b/tools/cli-package/process.mjs @@ -207,9 +207,16 @@ export function forwardInterrupts() { }; } +/** + * Run one tool child in its own process group (POSIX) and settle when it closed. Past `timeoutMs` its + * group (on Windows its whole tree) is killed. On POSIX with a positive `graceMs` the group first gets + * SIGTERM and `graceMs` more, so a child that stops its own detached children and removes its private + * directories on an interrupt (the producer, the consumer) does so before anything is killed. + */ export function runCommand(command, args, options = {}) { - const { timeoutMs = 300_000, maxBytes = 8 * 1024 * 1024, ...spawnOptions } = options; + const { timeoutMs = 300_000, graceMs = 0, maxBytes = 8 * 1024 * 1024, ...spawnOptions } = options; assert.ok(Number.isSafeInteger(timeoutMs) && timeoutMs > 0); + assert.ok(Number.isSafeInteger(graceMs) && graceMs >= 0); assert.ok(Number.isSafeInteger(maxBytes) && maxBytes > 0); return new Promise((resolve, reject) => { const grouped = process.platform !== 'win32'; @@ -225,7 +232,8 @@ export function runCommand(command, args, options = {}) { fault, closed = false, killer, - killerTimer; + killerTimer, + graceTimer; const output = { stdout: [], stderr: [] }; function stop() { if (closed || child.pid === undefined) return; @@ -254,7 +262,16 @@ export function runCommand(command, args, options = {}) { } const timer = setTimeout(() => { fault ??= new Error('PACKED_CLI_CHILD_TIMEOUT'); - stop(); + if (graceMs === 0 || !grouped || child.pid === undefined) { + stop(); + return; + } + try { + process.kill(-child.pid, 'SIGTERM'); + } catch (error) { + if (error.code !== 'ESRCH') fault ??= error; + } + graceTimer = setTimeout(stop, graceMs); }, timeoutMs); for (const sink of ['stdout', 'stderr']) child[sink].on('data', (chunk) => { @@ -271,6 +288,7 @@ export function runCommand(command, args, options = {}) { closed = true; liveChildren.delete(live); clearTimeout(timer); + clearTimeout(graceTimer); if (killer && killer.exitCode === null && killer.signalCode === null) await new Promise((done) => killer.once('close', done)); clearTimeout(killerTimer); diff --git a/tools/cli-package/process.test.mjs b/tools/cli-package/process.test.mjs index 28d43d89..e5091920 100644 --- a/tools/cli-package/process.test.mjs +++ b/tools/cli-package/process.test.mjs @@ -364,3 +364,65 @@ test( } }, ); + +test( + 'past its deadline a child is interrupted first, stops its own children and then settles', + { skip: process.platform === 'win32' }, + async () => { + const root = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-graceful-deadline-'))); + const marker = join(root, 'cleaned'); + const pidFile = join(root, 'grandchild.pid'); + try { + // A child that handles the interrupt as the producer does, around a detached grandchild. + const child = ` +import { writeFileSync } from 'node:fs'; +import { exitOnInterrupt, isolatedEnvironment, runCommand } from ${JSON.stringify(new URL('./process.mjs', import.meta.url).href)}; +await exitOnInterrupt( + () => writeFileSync(${JSON.stringify(marker)}, 'cleaned'), + () => runCommand(process.execPath, ['-e', ${JSON.stringify( + `require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, String(process.pid)); setInterval(() => {}, 1000);`, + )}], { env: isolatedEnvironment(${JSON.stringify(root)}), timeoutMs: 60_000 }), +); +`; + const started = Date.now(); + await assert.rejects( + runCommand(process.execPath, ['--input-type=module', '-e', child], { + timeoutMs: 3000, + graceMs: 20_000, + }), + /PACKED_CLI_CHILD_TIMEOUT/u, + ); + // It settled when the child exited on the interrupt, long before the grace ran out. + assert.ok(Date.now() - started < 15_000); + assert.equal(readFileSync(marker, 'utf8'), 'cleaned'); + const grandchild = Number(readFileSync(pidFile, 'utf8')); + for (let waited = 0; alive(grandchild); waited += 25) { + assert.ok(waited < 5_000, 'the grandchild outlived the cleanup of its parent'); + await delay(25); + } + } finally { + // A regressed deadline leaves the detached grandchild running: it never outlives the test. + const pid = existsSync(pidFile) ? Number(readFileSync(pidFile, 'utf8')) : NaN; + if (Number.isSafeInteger(pid) && pid > 0 && alive(pid)) process.kill(pid, 'SIGKILL'); + rmSync(root, { recursive: true, force: true }); + } + }, +); + +test( + 'a child that ignores the interrupt is killed once the grace runs out', + { skip: process.platform === 'win32' }, + async () => { + const started = Date.now(); + await assert.rejects( + runCommand( + process.execPath, + ['-e', "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);"], + { timeoutMs: 500, graceMs: 1000 }, + ), + (error) => + /PACKED_CLI_CHILD_TIMEOUT/u.test(error.message) && error.result.signal === 'SIGKILL', + ); + assert.ok(Date.now() - started >= 1500); + }, +); diff --git a/tools/cli-package/produce.test.mjs b/tools/cli-package/produce.test.mjs index 84a077dc..e24459ef 100644 --- a/tools/cli-package/produce.test.mjs +++ b/tools/cli-package/produce.test.mjs @@ -48,8 +48,9 @@ function ports({ plantCache = false, failEmit = false, dirty = false } = {}) { seen.pnpm = options; return { stdout: '9.12.3\n', stderr: '' }; }, - excludesFile: async ({ repository }) => { + excludesFile: async ({ repository, environment }) => { seen.excludesRepository = repository; + seen.excludesEnvironment = environment; return '/developer/excludes'; }, prepare: async (options) => { @@ -115,13 +116,18 @@ test('the producer builds and seals only the owned candidate, never the original } }); Object.assign(process.env, PLANTED); + const environment = { HOME: '/home/someone', NPM_TOKEN: 'environment-npm-token' }; const result = await producePackage({ repository, destination, - environment: { HOME: '/home/someone', NPM_TOKEN: 'environment-npm-token' }, + environment, temporary, ports: fakes, }); + // The developer's global excludes and dependency store are looked up in the caller's own + // environment, never in the isolated build environment. + assert.equal(seen.excludesEnvironment, environment); + assert.equal(seen.prepare.environment, environment); assert.deepEqual(result.identity, { source: SHA, dirty: false, files: 7 }); // The whole build — pnpm, the candidate's preparation and the emit with its pack — runs in the // isolated environment of the private build home, never with the caller's own variables. From 1f24777283197965579b71b76e0167f05cf9d3c4 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Fri, 23 Oct 2026 23:22:51 +0300 Subject: [PATCH 34/97] fix(repo): address the twelfth Step 9a.1 review round Round 12 (at 78623964) reported seven verified findings (five distinct), one medium; its plan, security and correctness lenses were clean. - Git reading the checkout gets the literal /dev/null as its null device on every platform. Node's Windows spelling (\\.\nul) is refused by Git for Windows, so every isolated Git call would have exited on a Windows host. - The consumer stage's deadline grace is tested, and the ignored-interrupt grace test is bounded and cleans up after itself. - packaging.md states the store-discovery exception; the register records round 12, the correction that rounds 3-11 skipped the filesystem and secrets families although both capture package.json, and the maintainer's 2026-10-23 acceptance ruling. Gates: lint:tools, typecheck:tools, the full pnpm run ci with every causal-control family, the real producer and consumer in smoke:cli-package, and the acceptance tools under CI=true GITHUB_ACTIONS=true; a mutation run of three mutants over the corrections kills all of them. Refs: ADR-0111, ADR-0051 Co-Authored-By: Claude Opus 5.5 --- docs/reference/cli/packaging.md | 9 +-- .../phase-2.5.5-hardening-and-remediation.md | 10 ++-- tools/cli-package/check.test.mjs | 55 +++++++++++++++++++ tools/cli-package/consume.mjs | 1 + tools/cli-package/process.mjs | 13 ++++- tools/cli-package/process.test.mjs | 45 ++++++++++----- 6 files changed, 109 insertions(+), 24 deletions(-) diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 8a3964df..53bc2e63 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -245,10 +245,11 @@ describes the implemented mechanism; independent acceptance is recorded in the changed during the copy. A link, as a listed file or as any parent directory of one below the checkout, refuses the copy, so a tracked directory replaced by an ignored link cannot bring in bytes from outside the checkout; so does an untracked nested repository, - which Git lists only as its directory. Every pnpm call runs in the private home's - isolated environment, but a corepack-managed pnpm runs from the developer's own corepack - cache (`COREPACK_HOME`, as corepack computes it), never from the temporary directory, - where another local user could plant a module on its lookup path. + which Git lists only as its directory. Every build and install pnpm call runs in the + private home's isolated environment (only the store discovery below also sees the + caller's store-location variables), and a corepack-managed pnpm runs from the developer's + own corepack cache (`COREPACK_HOME`, as corepack computes it), never from the temporary + directory, where another local user could plant a module on its lookup path. Ignored output, caches, dependencies and private notes never enter it. The locked closure is installed there offline, frozen and with `--ignore-scripts` from the existing content-addressed store, discovered with pnpm's store-location variables diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 732341e0..a92ab886 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -9393,7 +9393,7 @@ plan amendment (row PR95) and the recorded splits; ID coverage is unchanged apar | Area | Ruling | | --- | --- | -| Acceptance | Each increment keeps the existing rule: implement, author gates, commit, then NEW independent complete cumulative review rounds until two consecutive rounds are clean after the last material fix. Comment- or documentation-only nits do not reset the count. Security-touching increments add a separate scoped sitting: PR95, 9a.1, 9a.2, 9b, 9c, 9d (whole Step 9 custody), 10a, 10c, 11b, 13c, 14b, 15c, 16a, 16b, 16c (whole Step 16), 18 and 19b (whole wave). PR95 instead follows the maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)). | +| Acceptance | Each increment keeps the existing rule: implement, author gates, commit, then NEW independent complete cumulative review rounds until two consecutive rounds are clean after the last material fix. Comment- or documentation-only nits do not reset the count. Security-touching increments add a separate scoped sitting: PR95, 9a.1, 9a.2, 9b, 9c, 9d (whole Step 9 custody), 10a, 10c, 11b, 13c, 14b, 15c, 16a, 16b, 16c (whole Step 16), 18 and 19b (whole wave). PR95 instead follows the maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)). **Amended 2026-10-23 (maintainer, after Step 9a.1's twelfth round):** from Step 9a.1's round 13 on, every increment is accepted after one NEW independent complete cumulative round that reports no blocker, high or medium finding and no regression; that round's low findings are recorded in [deferred-tasks.md](../deferred-tasks.md) with their file, a minimal example and a re-check note rather than fixed in the loop, and nits are optional. A high or medium finding is still corrected and followed by a new round, and the separate security sittings above are unchanged. | | Publication | `development` is pushed after each accepted increment, so its actual CI (including the four same-tarball platform consumers) is observed per increment. The Wave 3 PR follows Step 19. | | Plan shape | PR95 + 9a.1, 9a.2, 9b, 9c, 9d + 10a–c + 11a–b + 12a–b + 13a–c + 14a–b + 15a–c + 16a–c + 17a–b + 18 + 19a–b (29 increments). Each split is recorded before its step starts. | | Reassignments | G54 moves from row 17 to 11a; #311's implementation moves to 10b and its certification stays in 17a; G36/G46 are re-scoped in 10a to the branch that replaces the raw-stack branch and are certified in 17b; promotion of the Steps 6–10 commands.md sections moves from Step 19 to 10c. | @@ -9592,10 +9592,12 @@ adversarially verified; the detailed reports stay local and Git-ignored. | 9 | `08f296cd` | NONCLEAN (plan, security, correctness and integration lenses clean) | 4 (3 low, 1 nit) | a failed production printed its build children's captured output word for word, and esbuild quotes a malformed upstream map in its warning, contrary to the rule that maps are excluded from every diagnostic: the producer's entry point now prints only prepack's own fixed `✗ CLI …` lines and one fixed code (`production-failure.mjs`), tested through the real entry point with a quoted-map marker. The local check's first stage guard gained a killing test (an interrupt during its own discovery), and the real-checkout excludes test no longer reads the host's system Git configuration | | 10 | `954d6a62` | NONCLEAN (plan, correctness, integration and critic clean) | 4 (3 low, 1 nit) | with a corepack-managed pnpm, the producer's pnpm ran from the isolated build home under the shared temporary directory, so its optional module lookups reached a `node_modules` there that another local user could create (present since Step 7b; shared-`/tmp` POSIX hosts only, never CI): isolated children now carry the developer's own corepack cache (`COREPACK_HOME`, computed as corepack computes it), and a test plants such a module and shows it never runs. A failed removal during an interrupt escaped the signal handler with a raw stack and exit 1: each cleanup step is now attempted on its own, a failure leaves one fixed line, the exit code stays the signal's, and an ordinary failed removal no longer skips the other directory. A source root's sibling sharing its prefix gained a killing test, and an untracked nested repository refuses under its own code | | 11 | `d890abf0` | NONCLEAN (plan, security, correctness and integration lenses clean) | 4 (1 medium, 3 low) | nothing pinned that the producer looks up the developer's global excludes in the caller's own environment rather than the isolated build environment, which would have dropped them silently: the composition test now requires the very environment object. The local check's deadline killed only a stage's own process group, so a producer's detached build child could recreate the removed root with a map: a stage past its deadline is now interrupted first (`runCommand`'s `graceMs`) and killed only after a grace period, tested with a late-writing build child. The check's wait for an interrupt caught during its own root removal gained a killing test, and `dependencyStore`'s docblock says where corepack runs | +| 12 | `78623964` | NONCLEAN (plan, security and correctness lenses clean) | 7 (1 medium, 2 low, 4 nits; 5 distinct) | Git reading the checkout got Node's null device, `\\.\nul` on Windows, which Git for Windows refuses (it maps only `nul` and the literal `/dev/null`), so every isolated Git call would exit on a Windows host — the local producer and a local consumer; the CI producer runs on Linux and every CI consumer names its source: it now gets the literal `/dev/null`. The consumer stage's deadline grace and a bounded grace test gained killing tests, and the store discovery's exception is stated. The author gate statements of rounds 3–11 said the filesystem and secrets families' captured sources were unchanged, but both capture the root `package.json`, which round 3 changed (one acceptance-tools entry); this round runs the full `pnpm run ci`, every family included | -**Remaining gates.** The increment is not independently accepted. Two NEW consecutive clean complete -cumulative rounds after the last material fix and a separate security sitting remain (the Acceptance -row of the 2026-10-19 rulings). Then `development` is pushed and its CI observed, and the PR #95 +**Remaining gates.** The increment is not independently accepted. Under the Acceptance row as amended +on 2026-10-23, one NEW complete cumulative round with no blocker, high or medium finding and no +regression, its security lens serving as the separate security sitting, remains; that round's low +findings go to deferred-tasks.md. Then `development` is pushed and its CI observed, and the PR #95 tooling nits in [deferred-tasks.md](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18) close. diff --git a/tools/cli-package/check.test.mjs b/tools/cli-package/check.test.mjs index 636d1d5f..2c6e89be 100644 --- a/tools/cli-package/check.test.mjs +++ b/tools/cli-package/check.test.mjs @@ -506,3 +506,58 @@ helper.unref(); } }, ); + +test( + 'a consumer past its deadline is interrupted first too, so its own children never outlive the check', + { skip: process.platform === 'win32' }, + async () => { + // The real consumer lifetime around a detached child that would write into the check root later. + const lateConsumer = (markers) => ` +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { completeConsumer } from './consumer-lifetime.mjs'; +import { isolatedEnvironment, runCommand } from './process.mjs'; +const reports = process.argv[3]; +mkdirSync(reports, { recursive: true }); +const home = mkdtempSync(join(tmpdir(), 'relavium-package-consumer-')); +const late = join(tmpdir(), 'late'); +await completeConsumer({ + supportRoot: reports, + exercise: () => + runCommand(process.execPath, ['-e', ${JSON.stringify( + `require('node:fs').writeFileSync(${JSON.stringify(join(markers, 'grandchild.pid'))}, String(process.pid));`, + )} + 'setTimeout(() => { require("node:fs").mkdirSync(' + JSON.stringify(late) + ', { recursive: true }); require("node:fs").writeFileSync(' + JSON.stringify(join(late, 'installed.log')) + ', "late"); }, 3000); setInterval(() => {}, 1000);'], { + env: isolatedEnvironment(home), + timeoutMs: 60_000, + }), + cleanup: () => { + rmSync(home, { recursive: true, force: true, maxRetries: 3 }); + writeFileSync(${JSON.stringify(join(markers, 'consume-cleaned'))}, 'cleaned'); + }, + cleanupFailed: () => {}, + publish: () => {}, +}); +`; + const run = check({ + env: { RELAVIUM_LOCAL_CHECK_DEADLINE_MS: '1500' }, + pack: RECORDING_PRODUCER, + consume: lateConsumer, + }); + try { + const grandchild = Number(await waitForFile(join(run.markers, 'grandchild.pid'))); + const [checkRoot] = checkRoots(run.temporary); + assert.ok(checkRoot !== undefined); + const outcome = await exitOf(run); + assert.equal(outcome.signal, null); + assert.equal(outcome.code, 1); + assert.equal(outcome.stdout, ''); + assert.equal(readFileSync(join(run.markers, 'consume-cleaned'), 'utf8'), 'cleaned'); + assert.equal(alive(grandchild), false); + await delay(3500); + assert.equal(existsSync(join(run.temporary, checkRoot)), false); + } finally { + stopAll(run); + } + }, +); diff --git a/tools/cli-package/consume.mjs b/tools/cli-package/consume.mjs index 930c1d1d..bd42a21f 100644 --- a/tools/cli-package/consume.mjs +++ b/tools/cli-package/consume.mjs @@ -41,6 +41,7 @@ const receipt = JSON.parse( readOrdinaryFile(join(artifactDirectory, 'package-test-receipt.json'), 512 * 1024), ); const origin = originFromEnvironment(process.env); +// CI always names the source; only a local run asks Git. const source = process.env.RELAVIUM_PACKAGE_SOURCE_SHA ?? ( diff --git a/tools/cli-package/process.mjs b/tools/cli-package/process.mjs index 66af623d..886a21aa 100644 --- a/tools/cli-package/process.mjs +++ b/tools/cli-package/process.mjs @@ -3,7 +3,7 @@ import assert from 'node:assert/strict'; import { spawn, spawnSync } from 'node:child_process'; import { Buffer } from 'node:buffer'; import { existsSync, lstatSync, readFileSync, realpathSync, statSync } from 'node:fs'; -import { constants, devNull, homedir } from 'node:os'; +import { constants, homedir } from 'node:os'; import { delimiter, extname, isAbsolute, join, resolve } from 'node:path'; import { clearTimeout, setImmediate, setTimeout } from 'node:timers'; @@ -50,6 +50,13 @@ export function isolatedEnvironment(home) { return env; } +/** + * The null device as Git names it on every platform. Git for Windows maps only `nul` and this literal + * to the device; Node's `os.devNull` there (`\\.\nul`) reaches the C runtime, which refuses it, and + * Git then exits rather than read no configuration. + */ +const GIT_NULL_DEVICE = '/dev/null'; + /** * Git with no global or system configuration, for reading the original checkout: its own `.git/config` * and ignore rules still apply and a caller passes the developer's global excludes explicitly, but no @@ -59,9 +66,9 @@ export function isolatedEnvironment(home) { */ export function isolatedGitEnvironment() { return { - ...isolatedEnvironment(devNull), + ...isolatedEnvironment(GIT_NULL_DEVICE), GIT_CONFIG_NOSYSTEM: '1', - GIT_CONFIG_GLOBAL: devNull, + GIT_CONFIG_GLOBAL: GIT_NULL_DEVICE, GIT_NO_REPLACE_OBJECTS: '1', }; } diff --git a/tools/cli-package/process.test.mjs b/tools/cli-package/process.test.mjs index e5091920..505f889a 100644 --- a/tools/cli-package/process.test.mjs +++ b/tools/cli-package/process.test.mjs @@ -11,7 +11,7 @@ import { rmSync, realpathSync, } from 'node:fs'; -import { devNull, homedir, tmpdir } from 'node:os'; +import { homedir, tmpdir } from 'node:os'; import { join, resolve, sep } from 'node:path'; import { fileURLToPath } from 'node:url'; import { @@ -28,12 +28,13 @@ import { test('Git reading the original checkout gets no home, global or system configuration nor replace refs', () => { const env = isolatedGitEnvironment(); assert.deepEqual(env, { - ...isolatedEnvironment(devNull), + ...isolatedEnvironment('/dev/null'), GIT_CONFIG_NOSYSTEM: '1', - GIT_CONFIG_GLOBAL: devNull, + GIT_CONFIG_GLOBAL: '/dev/null', GIT_NO_REPLACE_OBJECTS: '1', }); - for (const key of ['HOME', 'USERPROFILE', 'XDG_CONFIG_HOME']) assert.equal(env[key], devNull); + // The literal Git maps to the null device on every platform, never Node's Windows `\\.\nul`. + for (const key of ['HOME', 'USERPROFILE', 'XDG_CONFIG_HOME']) assert.equal(env[key], '/dev/null'); }); test('installer receives an explicit credential/config/preload-free environment', () => { @@ -412,17 +413,35 @@ await exitOnInterrupt( test( 'a child that ignores the interrupt is killed once the grace runs out', { skip: process.platform === 'win32' }, - async () => { + async (t) => { + const root = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-ignored-interrupt-'))); + const pidFile = join(root, 'child.pid'); + // A regressed grace leaves the child running: the bound fails the test and this stops it. + t.after(() => { + const pid = existsSync(pidFile) ? Number(readFileSync(pidFile, 'utf8')) : NaN; + if (Number.isSafeInteger(pid) && pid > 0 && alive(pid)) process.kill(pid, 'SIGKILL'); + rmSync(root, { recursive: true, force: true }); + }); const started = Date.now(); - await assert.rejects( - runCommand( - process.execPath, - ['-e', "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);"], - { timeoutMs: 500, graceMs: 1000 }, - ), - (error) => - /PACKED_CLI_CHILD_TIMEOUT/u.test(error.message) && error.result.signal === 'SIGKILL', + const settled = runCommand( + process.execPath, + [ + '-e', + `require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, String(process.pid));` + + "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);", + ], + { timeoutMs: 500, graceMs: 1000 }, + ).then( + () => 'resolved', + (error) => error, ); + const outcome = await Promise.race([ + settled, + delay(30_000, undefined, { ref: false }).then(() => 'still running'), + ]); + assert.ok(outcome instanceof Error, String(outcome)); + assert.match(outcome.message, /PACKED_CLI_CHILD_TIMEOUT/u); + assert.equal(outcome.result.signal, 'SIGKILL'); assert.ok(Date.now() - started >= 1500); }, ); From cf97b7f15a70969b4f3fde16e56aebaf90da27bb Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sat, 24 Oct 2026 03:44:57 +0300 Subject: [PATCH 35/97] fix(repo): close the Step 9a.1 internal rigor pass Before round 13, an internal pass over the whole increment at 1f247772 (five mutation sweeps plus claims, platform and security lenses) reported 84 raw findings: 6 medium, 69 low, 9 nits. - runCommand's deadline grace now kills the whole group when the leader exits on SIGTERM first, so a member that ignores SIGTERM cannot outlive it. - Git reading the checkout reads no system attributes and names its attributes and excludes files explicitly, so no default location applies, including the Windows locations under the null-device home. - The excludes lookup no longer runs Git from the checkout when the caller has no PATH. It sees command-scope GIT_CONFIG_* configuration, and store discovery forwards npm_config_store_dir. - The candidate copy keeps only the owner's execute bit, as Git does. - The consumer's placement check compares native real paths. - The producer removes the private maps first. An ordinary failure keeps its primary error and empties the artifact directory. - Finalizer refusals echo only validated, repository-relative subjects. - One unremovable bootstrap fixture no longer keeps the others in place. - Every wait for a spawned tool's exit in the interrupt tests is bounded. A regressed tool now fails its test instead of hanging it and leaving a detached descendant behind. - Each test gap gained a killing test. These include a composed sealPackage run, a behavioural tsup config check and exact finalizer bounds and subjects. They also cover the release guard's rule outcomes, the publish job's exact keys and the acceptance-tools list's completeness. - The docs scope forwarding and grace to POSIX, state the SIGHUP and shared-/tmp limits, and correct the register's scope list and security-sitting wording. - Five low findings that need a change of scope are recorded in deferred-tasks.md. Gates: the full pnpm run ci with every causal-control family, and the acceptance tools under CI=true GITHUB_ACTIONS=true (835 of 835). After the bounded waits were added (test files only; no causal-control family captures them), lint:test-isolation, lint:tools, typecheck:tools and the acceptance tools in both environments ran again. A re-run of the sweeps' 260 surviving mutants, plus new mutants of the corrections, kills every one except those the sweeps judged equivalent or defensive and the consumer and worker composition lines that no node test reaches (recorded in deferred-tasks.md). Refs: ADR-0111, ADR-0051 Co-Authored-By: Claude Opus 5.5 --- docs/reference/cli/packaging.md | 40 +- docs/roadmap/deferred-tasks.md | 48 ++ .../phase-2.5.5-hardening-and-remediation.md | 41 +- package.json | 2 +- tools/bundle-closure/closure.test.mjs | 78 +++- tools/ci/require-packed-cli.test.mjs | 41 +- tools/cli-bootstrap/child.mjs | 15 +- tools/cli-bootstrap/witness.test.mjs | 95 +++- tools/cli-package/README.md | 20 +- tools/cli-package/archive.test.mjs | 34 +- tools/cli-package/candidate.mjs | 56 ++- tools/cli-package/candidate.test.mjs | 293 ++++++++++-- tools/cli-package/check.test.mjs | 80 +++- tools/cli-package/child-exit.mjs | 24 + tools/cli-package/consume.mjs | 16 +- tools/cli-package/consumer-lifetime.test.mjs | 69 ++- tools/cli-package/finalize.mjs | 32 +- tools/cli-package/finalize.test.mjs | 421 ++++++++++++++++-- tools/cli-package/paths.mjs | 13 +- tools/cli-package/paths.test.mjs | 39 ++ tools/cli-package/pipeline.test.mjs | 7 + tools/cli-package/prepack.test.mjs | 72 ++- tools/cli-package/process.mjs | 29 +- tools/cli-package/process.test.mjs | 183 +++++++- tools/cli-package/produce.mjs | 108 +++-- tools/cli-package/produce.test.mjs | 336 ++++++++++++-- tools/cli-package/release-guard.test.mjs | 28 ++ tools/cli-package/seal.test.mjs | 247 +++++++++- tools/cli-package/smoke/README.md | 3 +- tools/cli-package/smoke/index.mjs | 11 +- tools/cli-package/smoke/native-process.mjs | 26 +- .../cli-package/smoke/native-process.test.mjs | 50 ++- tools/cli-package/smoke/worker.mjs | 5 +- tools/cli-package/ustar-fixture.mjs | 42 ++ 34 files changed, 2319 insertions(+), 285 deletions(-) create mode 100644 tools/cli-package/child-exit.mjs create mode 100644 tools/cli-package/paths.test.mjs create mode 100644 tools/cli-package/ustar-fixture.mjs diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 53bc2e63..1d668e2b 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -240,9 +240,10 @@ describes the implemented mechanism; independent acceptance is recorded in the tracked files Git is told not to check (skip-worktree or assume-unchanged, which make the identity `dirty` whatever their bytes), copies every file Git would ship (tracked plus untracked-but-not-ignored, minus deletions, by the repository's ignore rules and - the developer's global excludes file, passed explicitly; that Git reads no home, global - or system configuration and ignores replace refs) into a fresh owner-only tree, and refuses if any of the three - changed during the copy. A link, as a listed file or as any parent directory of one + the developer's global excludes file, as the developer's own Git finds it (also through + command-scope `GIT_CONFIG_*` configuration) and passed explicitly; that Git reads no home, + global or system configuration or attributes and ignores replace refs) into a fresh + owner-only tree, and refuses if any of the three changed during the copy. A link, as a listed file or as any parent directory of one below the checkout, refuses the copy, so a tracked directory replaced by an ignored link cannot bring in bytes from outside the checkout; so does an untracked nested repository, which Git lists only as its directory. Every build and install pnpm call runs in the @@ -252,9 +253,12 @@ describes the implemented mechanism; independent acceptance is recorded in the directory, where another local user could plant a module on its lookup path. Ignored output, caches, dependencies and private notes never enter it. The locked closure is installed there offline, frozen and with `--ignore-scripts` from the - existing content-addressed store, discovered with pnpm's store-location variables - only, and every `@relavium/*` link of the CLI and the five inlined engine packages must - resolve inside the tree. + existing content-addressed store, which pnpm discovers in the repository with only + `HOME`, `USERPROFILE`, `XDG_DATA_HOME`, `XDG_CONFIG_HOME`, `LOCALAPPDATA`, `APPDATA`, + `PNPM_HOME` and `npm_config_store_dir` of the caller's environment. Every `@relavium/*` + link of the CLI and the five inlined engine packages must resolve to that package inside + the tree, no other `@relavium/*` link is admitted, and the root `node_modules` must + resolve inside the tree too. - **Fresh upstream maps.** The five engine packages compile in order with the installed compiler's `--sourceMap true --inlineSources false --incremental false` overrides after their `dist` and incremental files are removed. No Turbo or other build cache is @@ -284,6 +288,9 @@ describes the implemented mechanism; independent acceptance is recorded in the rewritten maps (repository-relative sources; mappings and names unchanged) and the metafile owner-only outside the package, re-runs the finite closure and the migration-copy check on the finalized files, and only then hashes into a private record. + A refusal names a fixed code and at most a fixed label or a repository-relative path that + has passed validation: a metafile string that escapes the repository or is not a plain + path is left out, and no map or source content is ever printed. - **Seal.** The seal takes the candidate root, the staging directory and the original checkout's identity explicitly and never resolves a repository from its own location. Public JS must be exactly the staged metafile's runtime outputs, byte-identical to the @@ -304,11 +311,22 @@ and the producer still exits with the signal's code. An interrupted consumer lik child processes, including the bootstrap witness's, and removes its installation, the witness's fixtures and the installed smoke's working directory; the native fixture targets the installed CLI starts in their own process groups end on their own once the smoke worker that owns their case is -gone. The local `pnpm smoke:cli-package` forwards an interrupt to the producer or consumer it started -rather than ending first, starts no later stage once one arrived, and exits with the signal's -code after removing its root; a stage past its deadline is interrupted (SIGTERM) first and killed -only after a grace period, so it stops its own build children before the root goes. A producer killed outright, a crash or a lost host can leave the -owner-only root in the temporary directory, and no secure erasure is claimed. "Owner-only" in this +gone. On POSIX the local `pnpm smoke:cli-package` forwards an interrupt to the producer or +consumer it started rather than ending first (on Windows they share the console and receive +Ctrl-C themselves), starts no later stage once one arrived, and exits with the signal's code after +removing its root. On POSIX a stage past its deadline is interrupted (SIGTERM) first and killed, +with its whole process group, only after a grace period, so it stops its own build children before +the root goes; on Windows its whole tree is killed at the deadline, and its private directories go +with the check's root. These Windows interrupt and deadline paths are not exercised by any CI job: +the tooling tests run on Linux only. SIGHUP aborts and cleans up like the other two signals even +under `nohup`, which these listeners replace; detach a long local run with `setsid` or a terminal +multiplexer instead. A producer killed outright, a crash or a lost host can leave the +owner-only root in the temporary directory, and no secure erasure is claimed. The local consumer +installs the CLI under the temporary directory, so on a host whose temporary directory other +local users can write to (a shared `/tmp`), a module another user plants in its `node_modules` +could satisfy an optional `require` of the installed CLI; CI runners are single-tenant and macOS's +per-user temporary directory is not shared. This is a named limit of the local consumer, owned +by [deferred-tasks.md](../../roadmap/deferred-tasks.md#wave-3-step-9a1--recorded-low-findings-2026-10-24). "Owner-only" in this section means mode 0700 for directories and 0600 for staged files, set — and for the staging directory verified together with this process's uid — where POSIX modes are observable. On Windows no mode evidence is claimed: the candidate root and staging inherit the temporary directory's ACL, diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index 2ecad1f1..ae1e7d47 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -568,6 +568,54 @@ the repository, so each entry names the input and the property to re-check by ha [effect-journal.md](../reference/shared-core/effect-journal.md#11-secrets-what-a-row-may-hold) §11. *(packages/core/src/tools/bounding.ts)* +## Wave 3 Step 9a.1 — recorded low findings (2026-10-24) + +The internal rigor pass over Step 9a.1 at `1f247772`, run before its round 13 under the +[2026-10-23 acceptance amendment](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-implementation-checkpoint--2026-10-22), +corrected every verified code defect and test gap it could close inside the increment. These low findings are +recorded instead: each needs a change of scope (a new CI leg, a refactor of a Step 7b script) or concerns a +local-only path no claim covers. **Trigger for every entry:** the next change to the file it names, or a +real run that shows the shape. + +- [ ] **The local consumer installs under the shared temporary directory.** *Low, unchanged since Step 7b.* + *(rigor pass, SEC-1/PLAT-6)* `consume.mjs` installs the packed CLI under `mkdtemp(tmpdir())`, so the + installed CLI's optional bare requires (for example `ws`'s `require('bufferutil')`, loaded by the + `provider list` smoke case) fall through to `/node_modules`, which another local user can create on + a POSIX host with a shared `/tmp`. CI runners are single-tenant and macOS's per-user temporary directory is + not shared; packaging.md names the limit. **Re-check:** install under a per-user parent no other user can + write (for example a 0700 directory in the user's cache directory), with a test that plants + `/node_modules/bufferutil` and shows it never runs, as round 10's corepack test does. + *(low · tools/cli-package/consume.mjs)* +- [ ] **An interrupt during the consumer's publication leaves its receipt.** *Low.* *(rigor pass, PLAT-4)* + `publish()` runs inside `completeConsumer`'s interrupt scope, and the interrupt path removes the fixtures, + smoke roots and installation but not `installed-consumer.json`, so a consumer interrupted while publishing + exits 130 beside a success receipt (the producer empties its artifact directory in the same case). A later + local run into the same report directory then fails on the `wx` write; CI uploads only on success. + **Re-check:** write the receipt under a temporary name and rename it after the interrupt scope returns, or + retract it on the interrupt path, with a test that interrupts during `publish`. + *(low · tools/cli-package/consumer-lifetime.mjs)* +- [ ] **Two producers can share one artifact directory.** *Nit.* *(rigor pass, PLAT-9)* Both pass the + emptiness check (`mkdirSync` recursive plus `readdirSync`), so one seal can read the other's tarball. It + needs user error: the local check always uses a fresh root. **Re-check:** require that the directory does + not exist yet and create it without `recursive`, or take an exclusive lock file first. + *(nit · tools/cli-package/produce.mjs)* +- [ ] **The consumer's and the smoke worker's composition has no node test.** *Low, unchanged since Step 7b.* + *(rigor pass, F6 and F7)* `consume.mjs` hands `completeConsumer` the `supportRoot` that + `runInstalledSmoke` uses and gives the local source lookup `isolatedGitEnvironment()`, and the worker picks + its `node_modules` owner with `installedModulesOwner` and its root with `createInstalledSmokeRoot`; each + line can be changed wrongly without a node test failing, and a CI consumer, never interrupted, would not + notice a wrong `supportRoot`. The helpers themselves are tested. **Re-check:** export the consumer's + composition with ports, as `produce.mjs` is, and assert that one `supportRoot` binding reaches both calls and + that the Git call's environment equals `isolatedGitEnvironment()`. + *(low · tools/cli-package/consume.mjs, tools/cli-package/smoke/worker.mjs)* +- [ ] **The Windows interrupt, deadline and mode branches are not exercised.** *Low.* *(rigor pass, F14 and + MCG-9)* `stopLiveCommands`' taskkill path, `runCommand`'s ungrouped deadline, `forwardInterrupts`' skip of + ungrouped children, `stopBoundedChildren`'s fallback and the inventory's `not-observable-windows` mode + evidence run only on Windows, and `pnpm test:acceptance-tools` runs only on Linux; packaging.md says so. + **Re-check:** a Windows leg of the interrupt-free lifecycle checks (a stage past its deadline is killed at + once with `graceMs > 0`) and of `tools/bundle-closure/closure.test.mjs`. + *(low · tools/cli-package/process.mjs, tools/cli-bootstrap/process.mjs, tools/bundle-closure/inventory.mjs)* + ## Phase 2.6.5 deferrals — `CR-93` and `CR-95` (long half), recorded 2026-09-06; `CR-70`, `CR-94` (remainder) and the in-turn overflow continuation, recorded 2026-09-14; the errored-or-aborted turn residual, the pre-upgrade free-page bytes and the estimator's calibration, recorded 2026-09-18 **Phase closure — 2026-10-11:** W8 merged to `main` in PR #92 and Phase 2.6.5 is complete diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index a92ab886..0eb96bc0 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -9538,14 +9538,17 @@ corrections. It changes `tools/cli-package` (the owned candidate tree `candidate.mjs`; the producer `produce.mjs` and its entry point `pack.mjs`; the prepack orchestrator `prepack.mjs`; the finalizer `finalize.mjs`; the seal; the archive and smoke fold-ins; the interim publication guard `release-guard.mjs` with its rule -`release-guard-core.mjs`), `tools/bundle-closure` (map-aware closure and a shared emitted inventory), +`release-guard-core.mjs`; the interrupt, corepack and Git-isolation substrate in `process.mjs`; the local +check `check.mjs`; the consumer lifetime `consume.mjs` and `consumer-lifetime.mjs`; `paths.mjs` and +`production-failure.mjs`), `tools/bundle-closure` (map-aware closure and a shared emitted inventory), `apps/cli` (the prepack script and the candidate-mode tsup selection), `.github/workflows/release.yml` (the guard before npm publication) and, through the review rounds, `tools/cli-bootstrap` (stopping the bootstrap witness's bounded children and removing its live fixtures for an interrupted consumer), the installed smoke's native fixtures and worker root, the `ci.yml` pack-step comment, the acceptance-tools list in `package.json` and `tools/ci/require-packed-cli.test.mjs`, with the [packaging contract](../../reference/cli/packaging.md#candidate-build-and-finalizer-implementation-boundary--step-9a1), -the release runbook, an ADR-0051 amendment and the tooling README. It adds no runtime dependency, and +the release runbook, an ADR-0051 amendment, the tooling and smoke READMEs, the CLI reference index and the +`CLAUDE.md`, `AGENTS.md`, `current.md` and `deferred-tasks.md` status lines. It adds no runtime dependency, and the receipt stays `compiled_package_test`; the contract section is the canonical description of the mechanism. @@ -9594,10 +9597,40 @@ adversarially verified; the detailed reports stay local and Git-ignored. | 11 | `d890abf0` | NONCLEAN (plan, security, correctness and integration lenses clean) | 4 (1 medium, 3 low) | nothing pinned that the producer looks up the developer's global excludes in the caller's own environment rather than the isolated build environment, which would have dropped them silently: the composition test now requires the very environment object. The local check's deadline killed only a stage's own process group, so a producer's detached build child could recreate the removed root with a map: a stage past its deadline is now interrupted first (`runCommand`'s `graceMs`) and killed only after a grace period, tested with a late-writing build child. The check's wait for an interrupt caught during its own root removal gained a killing test, and `dependencyStore`'s docblock says where corepack runs | | 12 | `78623964` | NONCLEAN (plan, security and correctness lenses clean) | 7 (1 medium, 2 low, 4 nits; 5 distinct) | Git reading the checkout got Node's null device, `\\.\nul` on Windows, which Git for Windows refuses (it maps only `nul` and the literal `/dev/null`), so every isolated Git call would exit on a Windows host — the local producer and a local consumer; the CI producer runs on Linux and every CI consumer names its source: it now gets the literal `/dev/null`. The consumer stage's deadline grace and a bounded grace test gained killing tests, and the store discovery's exception is stated. The author gate statements of rounds 3–11 said the filesystem and secrets families' captured sources were unchanged, but both capture the root `package.json`, which round 3 changed (one acceptance-tools entry); this round runs the full `pnpm run ci`, every family included | +**Internal rigor pass, 2026-10-24.** After the twelfth round the maintainer asked for maximum care before +any further round, so an internal read-only pass covered the whole increment at `1f247772` before round +13: five mutation sweeps (the candidate and paths, the finalizer, the producer and seal, the process and +interrupt lifetime, and the closure and guard; each mutant applied in place, run in its own process group +and restored by digest) and claims, platform and security lenses; the detailed record is local. It +reported 84 raw findings (6 medium, 69 low, 9 nits). Code defects corrected: past its deadline, a group +member that ignored SIGTERM outlived a leader that exited on it, so the group is now killed when the leader +closes inside the grace; the excludes lookup ran Git from the checkout when the caller had no `PATH`, and +missed command-scope `GIT_CONFIG_*` configuration; Git reading the checkout still read the system +attributes and, on Windows, default attribute and ignore locations under the null-device home, so it now +sets `GIT_ATTR_NOSYSTEM` and names both files; the store discovery dropped `npm_config_store_dir`; the +candidate copy treated a group or other execute bit as executable, unlike Git; the consumer compared its +placement by real paths that keep the given letter case; the producer removed the private maps last on an +interrupt, and an ordinary failure let a removal error replace the primary refusal and left a refused +tarball in the artifact directory; two finalizer refusals echoed an unvalidated metafile string as their +subject; and one bootstrap fixture that could not be removed stopped the others' removal. Each test gap +gained a killing test, among them a composed `sealPackage` run on a candidate that is not the repository, +a behavioural evaluation of the tsup configuration, exact finalizer bounds and refusal subjects, the +release guard's handling of a rule that returns, exits or throws a foreign error, the publish job's exact +keys, and the completeness of the acceptance-tools list. The 260 mutants that had survived the five sweeps, +and new mutants over the corrections, were then run against the corrected tests. All are killed except the +ones the sweeps judged equivalent or defensive and the consumer and worker composition lines that no node +test reaches by design; two of the old survivors needed further killing tests first (the producer's default +temporary directory, and a deadline interrupt that reaches every member of the process group, not only its +leader). One of those runs left a test's detached build stand-in behind under a mutant that hung the test, +so every wait for a spawned tool's exit in the interrupt tests is now bounded and a regressed tool fails its +test instead. Five low findings that need a change of scope are recorded in +[deferred-tasks.md](../deferred-tasks.md#wave-3-step-9a1--recorded-low-findings-2026-10-24). + **Remaining gates.** The increment is not independently accepted. Under the Acceptance row as amended on 2026-10-23, one NEW complete cumulative round with no blocker, high or medium finding and no -regression, its security lens serving as the separate security sitting, remains; that round's low -findings go to deferred-tasks.md. Then `development` is pushed and its CI observed, and the PR #95 +regression remains, together with the separate scoped security sitting the Acceptance row requires, at +the same head and with the same bar (it runs alongside that round as its own reviewer and report, as in +every round so far); their low findings go to deferred-tasks.md. Then `development` is pushed and its CI observed, and the PR #95 tooling nits in [deferred-tasks.md](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18) close. diff --git a/package.json b/package.json index 0d6dcf71..d7604427 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "smoke:budget-replay": "node tools/budget-replay-compat/check.mjs", "check:wire-mutations": "node tools/wire-conformance/check.mjs", "check:usage-mutations": "node tools/usage-coverage-controls/check.mjs", - "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/check.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", + "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/check.test.mjs tools/cli-package/paths.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", "check:fs-mutations": "node tools/fs-policy-controls/check.mjs", "check:secrets-mutations": "node tools/secrets-controls/check.mjs", "smoke:cli-bootstrap": "node tools/cli-bootstrap/check.mjs", diff --git a/tools/bundle-closure/closure.test.mjs b/tools/bundle-closure/closure.test.mjs index b195f85d..f8245562 100644 --- a/tools/bundle-closure/closure.test.mjs +++ b/tools/bundle-closure/closure.test.mjs @@ -425,6 +425,10 @@ test('JS byte drift, unexpected permissions and hashbang drift each refuse', () const path = entry ? 'dist/index.js' : lazyFile(value); value.files[path].mode = entry ? 0o644 : 0o755; refuses(value, 'OUTPUT_MODE'); + // A set-id or sticky bit is part of the mode, not ignored beside it. + const special = candidate(); + special.files[path].mode = entry ? 0o4755 : 0o2644; + refuses(special, 'OUTPUT_MODE'); const other = candidate(); setContents( other, @@ -609,6 +613,11 @@ for (const [label, mutate, code] of [ }), 'OUTPUT_PATH', ], + ...[-1, 1.5, '7'].map((bytes) => [ + `map metadata with malformed bytes ${JSON.stringify(bytes)}`, + (value) => (value.metafile.outputs['dist/index.js.map'].bytes = bytes), + 'METADATA_SHAPE', + ]), ['an unknown map mode', (value) => (value.maps = 'external'), 'MAP_MODE'], ]) { test(`staged closure refuses ${label}`, () => { @@ -619,13 +628,16 @@ for (const [label, mutate, code] of [ } test('runtime code may never link a map, in an ordinary build or a finalized candidate', () => { - for (const value of [candidate(), staged()]) { - setContents( - value, - 'dist/index.js', - `${value.files['dist/index.js'].contents}//# sourceMappingURL=index.js.map`, - ); - refuses(value, 'MAP_LINK'); + // Node honours both comment forms; V8 and DevTools also honour the legacy `@` marker. + for (const link of [ + '//# sourceMappingURL=index.js.map', + '//@ sourceMappingURL=index.js.map', + '/*# sourceMappingURL=index.js.map */', + ]) { + for (const value of [candidate(), staged()]) { + setContents(value, 'dist/index.js', `${value.files['dist/index.js'].contents}${link}`); + refuses(value, 'MAP_LINK'); + } } }); @@ -658,3 +670,55 @@ test('the emitted inventory refuses maps and unreviewed files and admits only na rmSync(join(pkg, path)); } }); + +test('the emitted inventory never lists dist through a link', () => { + const pkg = mkdtempSync(join(scratch, 'linked-dist-')); + mkdirSync(join(pkg, 'real-dist')); + writeFileSync(join(pkg, 'real-dist', 'index.js'), '#!/usr/bin/env node\n'); + symlinkSync( + join(pkg, 'real-dist'), + join(pkg, 'dist'), + process.platform === 'win32' ? 'junction' : 'dir', + ); + assert.throws( + () => emittedInventory(pkg), + (error) => error instanceof BundleClosureError && error.code === 'OUTPUT_KIND', + ); +}); + +test( + 'the emitted inventory records a special file named like a chunk without reading it', + { skip: process.platform === 'win32' }, + () => { + const pkg = mkdtempSync(join(scratch, 'fifo-')); + mkdirSync(join(pkg, 'dist')); + assert.equal(spawnSync('mkfifo', [join(pkg, 'dist', 'chunk.js')]).status, 0); + // Reading a FIFO would block until a writer appears, so the inventory runs in a bounded child. + const run = spawnSync( + process.execPath, + [ + '--input-type=module', + '--eval', + `import { emittedInventory } from ${JSON.stringify(new URL('inventory.mjs', import.meta.url).href)}; +process.stdout.write(JSON.stringify(emittedInventory(${JSON.stringify(pkg)})));`, + ], + { encoding: 'utf8', timeout: 10_000, killSignal: 'SIGKILL' }, + ); + assert.equal(run.signal, null, 'the inventory blocked on a FIFO'); + assert.equal(run.status, 0, run.stderr); + const { kind, contents } = JSON.parse(run.stdout)['dist/chunk.js']; + assert.deepEqual({ kind, contents }, { kind: 'other', contents: '' }); + }, +); + +test( + 'the emitted inventory reports set-id and sticky bits in the mode', + { skip: process.platform === 'win32' }, + () => { + const pkg = mkdtempSync(join(scratch, 'special-mode-')); + mkdirSync(join(pkg, 'dist')); + writeFileSync(join(pkg, 'dist', 'index.js'), '#!/usr/bin/env node\n'); + chmodSync(join(pkg, 'dist', 'index.js'), 0o4755); + assert.equal(emittedInventory(pkg)['dist/index.js'].mode, 0o4755); + }, +); diff --git a/tools/ci/require-packed-cli.test.mjs b/tools/ci/require-packed-cli.test.mjs index 6f443ebb..d9e22611 100644 --- a/tools/ci/require-packed-cli.test.mjs +++ b/tools/ci/require-packed-cli.test.mjs @@ -1,5 +1,5 @@ import assert from 'node:assert/strict'; -import { readFileSync } from 'node:fs'; +import { readFileSync, readdirSync } from 'node:fs'; import { createRequire } from 'node:module'; import { test } from 'node:test'; @@ -276,7 +276,8 @@ test('release keeps main ancestry, version, real-push publication and npm proven }); test('release publication is refused fail-closed until a verified release candidate exists', () => { - const steps = readWorkflow('release').jobs.publish.steps; + const workflow = readWorkflow('release'); + const steps = workflow.jobs.publish.steps; const checkout = steps.find((step) => step.uses === actions.checkout); assert.deepEqual(checkout.with, { 'persist-credentials': false, @@ -289,10 +290,21 @@ test('release publication is refused fail-closed until a verified release candid ); const publish = steps.find((step) => step.name === 'Publish'); assert.equal(guard.run, 'node tools/cli-package/release-guard.mjs ./artifact'); - // An unconditional, non-advisory guard runs after the download and before any publish. - assert.equal(guard.if, undefined); - assert.equal(guard['continue-on-error'], undefined); - assert.equal(guard.env, undefined); + // An unconditional, non-advisory guard runs after the download and before any publish, under the + // runner's default shell and directory: no condition, shell, environment or defaults at the step, + // job or workflow level, and no earlier `run` step that could write the job's environment. + assert.deepEqual(Object.keys(guard).sort(), ['name', 'run']); + assert.deepEqual(Object.keys(workflow.jobs.publish).sort(), [ + 'if', + 'name', + 'needs', + 'permissions', + 'runs-on', + 'steps', + 'timeout-minutes', + ]); + assert.deepEqual(Object.keys(workflow).sort(), ['concurrency', 'jobs', 'name', 'on']); + assert.ok(steps.slice(0, steps.indexOf(guard)).every((step) => step.run === undefined)); assert.ok( steps.indexOf(steps.find((step) => step.uses === actions.download)) < steps.indexOf(guard), ); @@ -428,3 +440,20 @@ for (const command of ['pnpm smoke:cli-bootstrap', 'pnpm smoke:budget-replay']) }); } } + +test('every CLI package, closure, bootstrap and CI tooling test runs in test:acceptance-tools', () => { + // The list is the only place CI runs these files, so an unlisted test would silently never run. + const manifest = JSON.parse(readFileSync(new URL('package.json', root), 'utf8')); + const listed = new Set(manifest.scripts['test:acceptance-tools'].split(' ')); + for (const directory of [ + 'tools/cli-package', + 'tools/cli-package/smoke', + 'tools/bundle-closure', + 'tools/cli-bootstrap', + 'tools/ci', + ]) { + for (const name of readdirSync(new URL(directory, root))) { + if (name.endsWith('.test.mjs')) assert.ok(listed.has(`${directory}/${name}`), name); + } + } +}); diff --git a/tools/cli-bootstrap/child.mjs b/tools/cli-bootstrap/child.mjs index 4207414d..ce044258 100644 --- a/tools/cli-bootstrap/child.mjs +++ b/tools/cli-bootstrap/child.mjs @@ -39,13 +39,22 @@ const liveFixtures = new Set(); /** * For a caller that an interrupt ends before `withFixture` can remove its own root: synchronously kill - * every bounded child, then remove every fixture root still in use. + * every bounded child, then remove every fixture root still in use. One root that cannot be removed + * neither keeps the children alive nor the other roots in place; the first failure is rethrown after. */ export function abandonFixtures() { stopBoundedChildren(); - // Retries absorb a killed child that was still finishing a write. - for (const root of liveFixtures) rmSync(root, { recursive: true, force: true, maxRetries: 3 }); + let failure; + for (const root of liveFixtures) { + try { + // Retries absorb a killed child that was still finishing a write. + rmSync(root, { recursive: true, force: true, maxRetries: 3 }); + } catch (error) { + failure ??= error; + } + } liveFixtures.clear(); + if (failure !== undefined) throw failure; } /** Each child receives a synthetic HOME/cwd, no ambient credentials or Node preload. */ diff --git a/tools/cli-bootstrap/witness.test.mjs b/tools/cli-bootstrap/witness.test.mjs index 8c5a9ff5..2de0eeb1 100644 --- a/tools/cli-bootstrap/witness.test.mjs +++ b/tools/cli-bootstrap/witness.test.mjs @@ -3,6 +3,7 @@ import { spawn } from 'node:child_process'; import { test } from 'node:test'; import { setTimeout as delay } from 'node:timers/promises'; import { + chmodSync, copyFileSync, existsSync, mkdirSync, @@ -16,6 +17,7 @@ import { import { tmpdir } from 'node:os'; import { join, relative } from 'node:path'; import { pathToFileURL } from 'node:url'; +import { exitWithin } from '../cli-package/child-exit.mjs'; import { withFixture, toEntryUrl } from './child.mjs'; import { assertWitness, importFailureCanary, servicePackages } from './policy.mjs'; import { spawnBounded, stopBoundedChildren } from './process.mjs'; @@ -469,9 +471,7 @@ await withFixture((fixture) => readdirSync(temporary).filter((name) => name.startsWith('relavium-bootstrap-')).length, 1, ); - const exited = new Promise((resolve) => - caller.once('exit', (code, signal) => resolve([code, signal])), - ); + const exited = exitWithin(caller); caller.kill('SIGINT'); assert.deepEqual(await exited, [130, null]); // The whole process group goes, not only the process spawnBounded started. @@ -489,6 +489,95 @@ await withFixture((fixture) => } }, ); +test( + 'abandoning kills a group that ignores SIGTERM and removes the other fixtures when one removal fails', + { skip: process.platform === 'win32' || process.getuid?.() === 0 }, + async (t) => { + const temporary = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-bootstrap-locked-'))); + const pidFiles = ['locked.pid', 'grandchild.pid', 'plain.pid'].map((name) => + join(temporary, name), + ); + const roots = () => + readdirSync(temporary).filter((name) => name.startsWith('relavium-bootstrap-')); + const url = (name) => JSON.stringify(new URL(name, import.meta.url).href); + const idle = (pidFile) => + `require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, String(process.pid));` + + `process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);`; + // The first control's child makes its fixture unremovable (a file in a read-only directory) + // and, like its grandchild, ignores SIGTERM; the second control's fixture is ordinary. + const locked = + `const fs = require('node:fs'); const path = require('node:path');` + + `const directory = path.join(process.env.TMPDIR, 'locked'); fs.mkdirSync(directory);` + + `fs.writeFileSync(path.join(directory, 'file'), 'x'); fs.chmodSync(directory, 0o500);` + + `require('node:child_process').spawn(process.execPath, ['-e', ${JSON.stringify( + idle(pidFiles[1]), + )}], { stdio: 'ignore' });` + + idle(pidFiles[0]); + const script = ` +import { writeSync } from 'node:fs'; +import { abandonFixtures, withFixture } from ${url('./child.mjs')}; +import { spawnBounded } from ${url('./process.mjs')}; +process.on('SIGINT', () => { + try { + abandonFixtures(); + } catch { + writeSync(2, 'ABANDON_FAILED\\n'); + } + process.exit(130); +}); +const control = (code) => (fixture) => + spawnBounded(['-e', code], { env: { TMPDIR: fixture.temporary } }, { timeoutMs: 60_000 }); +await Promise.all([ + withFixture(control(${JSON.stringify(locked)})), + withFixture(control(${JSON.stringify(idle(pidFiles[2]))})), +]); +`; + const caller = spawn(process.execPath, ['--input-type=module', '-e', script], { + env: { PATH: process.env.PATH, TMPDIR: temporary }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stderr = ''; + caller.stderr.on('data', (chunk) => (stderr += chunk)); + const alive = (pid) => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } + }; + const tree = []; + t.after(() => { + if (caller.exitCode === null && caller.signalCode === null) caller.kill('SIGKILL'); + for (const pid of tree) if (alive(pid)) process.kill(pid, 'SIGKILL'); + for (const name of roots()) { + const directory = join(temporary, name, 'temporary', 'locked'); + if (existsSync(directory)) chmodSync(directory, 0o700); + } + rmSync(temporary, { recursive: true, force: true }); + }); + for (const file of pidFiles) { + for (let waited = 0; !existsSync(file) || readFileSync(file, 'utf8') === ''; waited += 25) { + assert.ok(waited < 20_000, 'a bounded control never started'); + await delay(25); + } + tree.push(Number(readFileSync(file, 'utf8'))); + } + assert.equal(roots().length, 2); + const exited = exitWithin(caller); + caller.kill('SIGINT'); + assert.deepEqual(await exited, [130, null]); + for (let waited = 0; tree.some(alive); waited += 25) { + assert.ok(waited < 5_000, 'a bounded control outlived an abandonment whose removal failed'); + await delay(25); + } + // Only the unremovable fixture is left, and the failure was reported, not swallowed. + const left = roots(); + assert.equal(left.length, 1); + assert.equal(existsSync(join(temporary, left[0], 'temporary', 'locked')), true); + assert.equal(stderr, 'ABANDON_FAILED\n'); + }, +); test( 'a closed bounded child leaves nothing for an interrupted caller to kill', { skip: process.platform === 'win32' }, diff --git a/tools/cli-package/README.md b/tools/cli-package/README.md index e91ef897..8a10d530 100644 --- a/tools/cli-package/README.md +++ b/tools/cli-package/README.md @@ -17,7 +17,7 @@ pnpm smoke:cli-package The producer (`produce.mjs`, run by `pack.mjs`) never builds in the developer's workspace. It copies the files Git would ship, by the repository's ignore rules and the developer's -global excludes file (its Git reads no other home, global or system configuration), into a +global excludes file (its Git reads no other home, global or system configuration or attributes), into a fresh owner-only candidate tree (owner-only where POSIX modes are observable; on Windows it inherits the temporary directory's ACL, which is not verified), refusing if the original checkout's commit, status or set of files Git is told not to check changes during the copy (such a skip-worktree or assume-unchanged @@ -25,8 +25,8 @@ file makes the source `dirty`), and installs the locked closure there offline, f without lifecycle scripts from the existing store. In that tree it emits the five inlined dependency packages in order with the installed compiler, upstream maps and no build cache. Prepack (`prepack.mjs`) performs the sole final CLI build in candidate -mode and runs the finalizer (`finalize.mjs`), which validates every upstream and final -map, removes only the exact map-link trailers and stages maps, metafile and its hash +mode and runs the finalizer (`finalize.mjs`), which validates every consumed upstream map and every +final map, removes only the exact map-link trailers and stages maps, metafile and its hash record in a private directory outside the artifact. The runner observes one actual prepack build-start record and one finalization, checks the emitted graph, and independently inspects the produced gzip/USTAR bytes. The inspector never extracts @@ -45,10 +45,16 @@ private staging, also when the producer is interrupted (SIGINT, SIGTERM, SIGHUP) stops the build first and empties the artifact directory, even after the last build child exited. An interrupted consumer stops its children, the bootstrap witness's included, and removes its installation, the witness's fixtures and the installed smoke's working directory; the -installed CLI's detached native fixture targets end once the smoke worker that owns them is gone. `pnpm smoke:cli-package` forwards an interrupt to the producer or -consumer it started instead of ending before them and starts no later stage after one, and it -interrupts a stage past its deadline before it kills it, so the stage stops its own children; a -producer killed outright can leave its owner-only root in the temporary directory. A failed +installed CLI's detached native fixture targets end once the smoke worker that owns them is gone. +On POSIX `pnpm smoke:cli-package` forwards an interrupt to the producer or consumer it started +instead of ending before them, and it interrupts a stage past its deadline before it kills the +stage's process group, so the stage stops its own children; on Windows the stages receive Ctrl-C +from the shared console and a stage past its deadline is killed with its tree at once (no CI job +exercises these Windows paths). It starts no later stage after an interrupt. SIGHUP aborts and +cleans up even under `nohup`; use `setsid` or a terminal multiplexer to detach a long run. A +producer killed outright can leave its owner-only root in the temporary directory. The local +consumer installs under the temporary directory, so a shared `/tmp` is a named limit +(packaging.md). A failed production prints only prepack's own fixed `✗ CLI …` lines and one fixed code (`production-failure.mjs`), never a build child's captured output, which can quote a map. A corepack-managed pnpm always runs from the developer's own corepack cache, never from the shared diff --git a/tools/cli-package/archive.test.mjs b/tools/cli-package/archive.test.mjs index fd381714..1566b793 100644 --- a/tools/cli-package/archive.test.mjs +++ b/tools/cli-package/archive.test.mjs @@ -4,6 +4,7 @@ import { createHash } from 'node:crypto'; import { test } from 'node:test'; import { gzipSync } from 'node:zlib'; import { ARCHIVE_LIMITS, inspectPackageArchive, PackageArchiveError } from './archive.mjs'; +import { checksum, entry, ustar } from './ustar-fixture.mjs'; const expectedPackage = { name: 'relavium', @@ -30,40 +31,9 @@ function inventory() { ]); } -/** @param {Buffer} header */ -function checksum(header) { - header.fill(32, 148, 156); - header.write( - header - .reduce((sum, byte) => sum + byte, 0) - .toString(8) - .padStart(6, '0'), - 148, - ); - header[154] = 0; - header[155] = 32; -} - -/** @param {string} path @param {Uint8Array} bytes */ -function entry(path, bytes) { - const header = Buffer.alloc(512); - header.write(path, 0, 100); - for (const offset of [100, 108, 116, 329, 337]) header.write('000000 \0', offset); - header.write(path === 'package/dist/index.js' ? '000755' : '000644', 100); - header.write(bytes.length.toString(8).padStart(11, '0') + ' ', 124); - header.write('00000000000 ', 136); - header[156] = 48; - header.write('ustar\0' + '00', 257); - checksum(header); - return Buffer.concat([header, bytes, Buffer.alloc((512 - (bytes.length % 512)) % 512)]); -} - /** @param {Map} [files] */ function tar(files = inventory()) { - return Buffer.concat([ - ...[...files].map(([path, bytes]) => entry(path, bytes)), - Buffer.alloc(1024), - ]); + return ustar(files); } /** @param {Buffer} input @param {string} code @param {Map} [files] */ diff --git a/tools/cli-package/candidate.mjs b/tools/cli-package/candidate.mjs index b7821086..8ce20a01 100644 --- a/tools/cli-package/candidate.mjs +++ b/tools/cli-package/candidate.mjs @@ -22,7 +22,13 @@ import { isDeepStrictEqual } from 'node:util'; import { ENGINE_PACKAGES } from './finalize.mjs'; import { readOrdinaryFile } from './files.mjs'; import { within } from './paths.mjs'; -import { isolatedEnvironment, isolatedGitEnvironment, runCommand, runPnpm } from './process.mjs'; +import { + GIT_NULL_DEVICE, + isolatedEnvironment, + isolatedGitEnvironment, + runCommand, + runPnpm, +} from './process.mjs'; export const CANDIDATE_LIMITS = Object.freeze({ files: 20_000, @@ -39,6 +45,8 @@ const STORE_DISCOVERY_VARIABLES = Object.freeze([ 'LOCALAPPDATA', 'APPDATA', 'PNPM_HOME', + 'npm_config_store_dir', + 'NPM_CONFIG_STORE_DIR', ]); /** @param {string} output NUL-separated Git path list */ @@ -49,7 +57,12 @@ function gitPaths(output) { /** Names the developer's global Git excludes file for a producer whose own environment is isolated. */ export const GIT_EXCLUDES_ENV = 'RELAVIUM_GIT_EXCLUDES_FILE'; -/** Only the variables Git uses to locate its global configuration; never credentials or hooks. */ +/** + * Only the variables Git uses to locate its global configuration, plus the command-scope configuration + * the developer's Git takes from the environment (`GIT_CONFIG_COUNT` with its `GIT_CONFIG_KEY_n` and + * `GIT_CONFIG_VALUE_n`, and `GIT_CONFIG_PARAMETERS`); never credentials or hooks. They reach only the + * read-only lookup of `core.excludesFile`, which runs nothing. + */ const GIT_CONFIG_VARIABLES = Object.freeze([ 'HOME', 'USERPROFILE', @@ -57,9 +70,12 @@ const GIT_CONFIG_VARIABLES = Object.freeze([ 'GIT_CONFIG_GLOBAL', 'GIT_CONFIG_SYSTEM', 'GIT_CONFIG_NOSYSTEM', + 'GIT_CONFIG_COUNT', + 'GIT_CONFIG_PARAMETERS', 'SystemRoot', 'WINDIR', ]); +const GIT_CONFIG_ENTRY = /^GIT_CONFIG_(?:KEY|VALUE)_(?:0|[1-9][0-9]{0,3})$/u; /** * The developer's global Git excludes file, so that the candidate's isolated Git ignores what the @@ -79,10 +95,14 @@ export async function globalExcludesFile({ repository, environment, run = runCom assert.ok(isAbsolute(explicit) && existsSync(explicit), 'PACKED_CLI_GIT_EXCLUDES'); return explicit; } - const env = { PATH: environment.PATH ?? '' }; + // No PATH means no PATH: an empty one would make the spawn look the command up in the checkout. + const env = environment.PATH === undefined ? {} : { PATH: environment.PATH }; for (const key of GIT_CONFIG_VARIABLES) { if (environment[key] !== undefined) env[key] = environment[key]; } + for (const [key, value] of Object.entries(environment)) { + if (GIT_CONFIG_ENTRY.test(key) && value !== undefined) env[key] = value; + } let configured; try { configured = ( @@ -119,9 +139,18 @@ export function producerEnvironment(base, { store, excludes }) { }; } -/** @param {string | undefined} excludes @returns {string[]} */ -function excludesArguments(excludes) { - return excludes === undefined ? [] : ['-c', `core.excludesFile=${excludes}`]; +/** + * The attributes and excludes files a checkout-reading Git uses, named explicitly so that no default + * location applies — not even one Git derives from the null-device home on Windows. + * @param {string | undefined} excludes @returns {string[]} + */ +function readArguments(excludes) { + return [ + '-c', + `core.attributesFile=${GIT_NULL_DEVICE}`, + '-c', + `core.excludesFile=${excludes ?? GIT_NULL_DEVICE}`, + ]; } /** @@ -138,9 +167,9 @@ function uncheckedPaths(output) { /** * The source identity of the original checkout: its exact commit, its full porcelain status and the * tracked files Git is told not to check, so a candidate copied from a modified tree records `dirty` - * and a change to either during the copy refuses. A skip-worktree or assume-unchanged file counts as - * dirty whatever its bytes, since its copy is never compared with the commit. Untracked files are - * judged with the developer's global excludes (`globalExcludesFile`). + * and a change to any of the three during the copy refuses. A skip-worktree or assume-unchanged file + * counts as dirty whatever its bytes, since its copy is never compared with the commit. Untracked + * files are judged with the developer's global excludes (`globalExcludesFile`). * @param {string} repository @param {typeof runCommand} [run] @param {string} [excludes] */ export async function sourceIdentity(repository, run = runCommand, excludes = undefined) { @@ -152,7 +181,7 @@ export async function sourceIdentity(repository, run = runCommand, excludes = un const status = ( await run( 'git', - [...excludesArguments(excludes), 'status', '--porcelain=v1', '-z', '--untracked-files=all'], + [...readArguments(excludes), 'status', '--porcelain=v1', '-z', '--untracked-files=all'], options, ) ).stdout; @@ -180,7 +209,7 @@ export async function candidateFiles( await run( 'git', [ - ...excludesArguments(excludes), + ...readArguments(excludes), 'ls-files', '-z', '--cached', @@ -207,7 +236,7 @@ export async function candidateFiles( /** * Copy each listed ordinary file byte-for-byte into the empty `destination`. Links (also as a parent * directory of a listed path), special files, escaping or noncanonical paths and oversized trees - * refuse; only the executable bit is kept, as an + * refuse; only the owner's executable bit is kept, as Git records it, as an * exact 0755 or 0644 mode under any umask that lets the owner write (one that removes the owner's * write bit leaves the created directories unwritable, and the copy refuses before any build). * @param {{repository: string, destination: string, files: readonly string[], @@ -241,7 +270,8 @@ export function copyCandidateTree({ repository, destination, files, limits = CAN directories.add(parent); } const source = join(repository, ...parts); - const executable = (lstatSync(source).mode & 0o111) !== 0; + // Git records a file as executable by its owner's execute bit alone. + const executable = (lstatSync(source).mode & 0o100) !== 0; const bytes = readOrdinaryFile(source, limits.fileBytes); total += bytes.length; assert.ok(total <= limits.totalBytes, 'PACKED_CLI_CANDIDATE_BYTES'); diff --git a/tools/cli-package/candidate.test.mjs b/tools/cli-package/candidate.test.mjs index 9e746ade..64de9dfe 100644 --- a/tools/cli-package/candidate.test.mjs +++ b/tools/cli-package/candidate.test.mjs @@ -1,7 +1,10 @@ import assert from 'node:assert/strict'; import { chmodSync, + closeSync, existsSync, + ftruncateSync, + openSync, mkdirSync, mkdtempSync, readFileSync, @@ -14,7 +17,7 @@ import { writeFileSync, } from 'node:fs'; import { tmpdir } from 'node:os'; -import { dirname, join } from 'node:path'; +import { dirname, join, relative } from 'node:path'; import { after, test } from 'node:test'; import { isDeepStrictEqual } from 'node:util'; import { @@ -28,6 +31,7 @@ import { producerEnvironment, sourceIdentity, verifyCandidateLinks, + CANDIDATE_LIMITS, } from './candidate.mjs'; import { isolatedGitEnvironment, runCommand } from './process.mjs'; @@ -63,6 +67,9 @@ function git(answers) { } const SHA = 'a'.repeat(40); +/** The explicit attributes and excludes arguments of a checkout-reading status or listing. */ +const READ = (excludes = '/dev/null') => + `-c core.attributesFile=/dev/null -c core.excludesFile=${excludes} `; test('source identity records the exact commit and the complete porcelain status', async (t) => { // A credential in the caller's environment must not reach the isolated Git. @@ -74,7 +81,7 @@ test('source identity records the exact commit and the complete porcelain status }); const clean = git({ '--no-replace-objects rev-parse HEAD': `${SHA}\n`, - 'status --porcelain=v1 -z --untracked-files=all': '', + [`${READ()}status --porcelain=v1 -z --untracked-files=all`]: '', 'ls-files -v -z': 'H a.ts\0H b.ts\0', }); assert.deepEqual(await sourceIdentity('/repository', clean.run), { @@ -93,7 +100,7 @@ test('source identity records the exact commit and the complete porcelain status ); const modified = git({ '--no-replace-objects rev-parse HEAD': `${SHA}\n`, - 'status --porcelain=v1 -z --untracked-files=all': '?? notes.txt\0', + [`${READ()}status --porcelain=v1 -z --untracked-files=all`]: '?? notes.txt\0', 'ls-files -v -z': 'H a.ts\0', }); assert.equal((await sourceIdentity('/repository', modified.run)).dirty, true); @@ -101,7 +108,7 @@ test('source identity records the exact commit and the complete porcelain status // copied unchecked, so it counts as dirty even when the status is empty. const unchecked = git({ '--no-replace-objects rev-parse HEAD': `${SHA}\n`, - 'status --porcelain=v1 -z --untracked-files=all': '', + [`${READ()}status --porcelain=v1 -z --untracked-files=all`]: '', 'ls-files -v -z': 'H a.ts\0S sparse.ts\0h assumed.ts\0s both.ts\0M merged.ts\0', }); assert.deepEqual(await sourceIdentity('/repository', unchecked.run), { @@ -113,7 +120,7 @@ test('source identity records the exact commit and the complete porcelain status // The developer's global excludes file judges untracked files. const excluded = git({ '--no-replace-objects rev-parse HEAD': `${SHA}\n`, - '-c core.excludesFile=/home/someone/.config/git/ignore status --porcelain=v1 -z --untracked-files=all': + [`${READ('/home/someone/.config/git/ignore')}status --porcelain=v1 -z --untracked-files=all`]: '', 'ls-files -v -z': '', }); @@ -124,7 +131,7 @@ test('source identity records the exact commit and the complete porcelain status for (const head of ['', 'HEAD\n', `${SHA.slice(1)}\n`, `${SHA}${SHA}\n`]) { const malformed = git({ '--no-replace-objects rev-parse HEAD': head, - 'status --porcelain=v1 -z --untracked-files=all': '', + [`${READ()}status --porcelain=v1 -z --untracked-files=all`]: '', 'ls-files -v -z': '', }); await assert.rejects(sourceIdentity('/repository', malformed.run), /SOURCE_IDENTITY/u); @@ -133,24 +140,25 @@ test('source identity records the exact commit and the complete porcelain status test('candidate files are tracked plus unignored untracked files, minus deletions, sorted once', async () => { const listing = git({ - 'ls-files -z --cached --others --exclude-standard': 'b.ts\0a.ts\0gone.ts\0a.ts\0new.ts\0', + [`${READ()}ls-files -z --cached --others --exclude-standard`]: + 'b.ts\0a.ts\0gone.ts\0a.ts\0new.ts\0', 'ls-files -z --deleted': 'gone.ts\0', }); assert.deepEqual(await candidateFiles('/repository', listing.run), ['a.ts', 'b.ts', 'new.ts']); assert.ok(listing.calls.every((call) => isDeepStrictEqual(call.env, isolatedGitEnvironment()))); const empty = git({ - 'ls-files -z --cached --others --exclude-standard': '', + [`${READ()}ls-files -z --cached --others --exclude-standard`]: '', 'ls-files -z --deleted': '', }); await assert.rejects(candidateFiles('/repository', empty.run), /CANDIDATE_FILES/u); const excluded = git({ - '-c core.excludesFile=/ignore ls-files -z --cached --others --exclude-standard': 'a.ts\0', + [`${READ('/ignore')}ls-files -z --cached --others --exclude-standard`]: 'a.ts\0', 'ls-files -z --deleted': '', }); assert.deepEqual(await candidateFiles('/repository', excluded.run, '/ignore'), ['a.ts']); // An untracked nested repository is listed only as its directory and refuses by name. const nested = git({ - 'ls-files -z --cached --others --exclude-standard': 'a.ts\0vendor/clone/\0', + [`${READ()}ls-files -z --cached --others --exclude-standard`]: 'a.ts\0vendor/clone/\0', 'ls-files -z --deleted': '', }); await assert.rejects( @@ -165,7 +173,7 @@ test("the developer's global excludes file is located without the rest of the gl write(ignore, 'secret.local\n'); const calls = []; const unset = async (command, args, options) => { - calls.push({ args, env: options.env }); + calls.push({ args, env: options.env, cwd: options.cwd }); const error = new Error('PACKED_CLI_CHILD_FAILED: 1'); error.result = { code: 1, stdout: '', stderr: '' }; throw error; @@ -178,6 +186,11 @@ test("the developer's global excludes file is located without the rest of the gl GIT_CONFIG_GLOBAL: join(home, 'global-config'), GIT_CONFIG_SYSTEM: join(home, 'system-config'), GIT_CONFIG_NOSYSTEM: '1', + // Command-scope configuration the developer's Git takes from the environment. + GIT_CONFIG_COUNT: '1', + GIT_CONFIG_KEY_0: 'core.excludesFile', + GIT_CONFIG_VALUE_0: '/env/ignore', + GIT_CONFIG_PARAMETERS: "'core.quotepath'='false'", SystemRoot: 'C:\\Windows', WINDIR: 'C:\\Windows', }; @@ -187,6 +200,7 @@ test("the developer's global excludes file is located without the rest of the gl NPM_TOKEN: 'synthetic-secret', NODE_OPTIONS: '--require=/tmp/preload.cjs', GIT_DIR: '/elsewhere', + GIT_CONFIG_KEY_X: 'not an entry', }; assert.equal( await globalExcludesFile({ @@ -197,9 +211,50 @@ test("the developer's global excludes file is located without the rest of the gl ignore, ); assert.deepEqual(calls[0].args, ['config', '--path', '--get', 'core.excludesFile']); + // It reads the checkout's own configuration as the developer's Git there would. + assert.equal(calls[0].cwd, '/repository'); assert.deepEqual(calls[0].env, { PATH: '/usr/bin', ...locations, XDG_CONFIG_HOME: '' }); await globalExcludesFile({ repository: '/repository', environment, run: unset }); assert.deepEqual(calls[1].env, { PATH: '/usr/bin', ...locations }); + // With no PATH the lookup gets no PATH either, never an empty one that names the checkout. + await globalExcludesFile({ repository: '/repository', environment: { HOME: home }, run: unset }); + assert.equal('PATH' in calls.at(-1).env, false); + // Without HOME, Git's default uses USERPROFILE; with both, HOME wins. + const profile = fresh('profile'); + write(join(profile, '.config', 'git', 'ignore'), '*.profile\n'); + assert.equal( + await globalExcludesFile({ + repository: '/repository', + environment: { USERPROFILE: profile }, + run: unset, + }), + join(profile, '.config', 'git', 'ignore'), + ); + assert.equal( + await globalExcludesFile({ + repository: '/repository', + environment: { HOME: home, USERPROFILE: profile }, + run: unset, + }), + ignore, + ); + // An empty HOME names no default, not one relative to this process's working directory. + const cwd = process.cwd(); + const relativeDefault = fresh('relative-default'); + write(join(relativeDefault, '.config', 'git', 'ignore'), '*.relative\n'); + process.chdir(relativeDefault); + try { + assert.equal( + await globalExcludesFile({ + repository: '/repository', + environment: { HOME: '' }, + run: unset, + }), + undefined, + ); + } finally { + process.chdir(cwd); + } // Git's default prefers XDG_CONFIG_HOME, and a missing file means no global excludes. const xdg = fresh('xdg'); write(join(xdg, 'git', 'ignore'), '*.local\n'); @@ -241,6 +296,15 @@ test("the developer's global excludes file is located without the rest of the gl }), join(repository, 'local-ignore'), ); + // A configured path that is a directory names no excludes file. + assert.equal( + await globalExcludesFile({ + repository: '/repository', + environment: { HOME: home }, + run: answer(`${fresh('directory-excludes')}\n`), + }), + undefined, + ); // A setting present but empty disables Git's global excludes: the default file no longer applies. assert.equal( await globalExcludesFile({ @@ -268,7 +332,8 @@ test("the developer's global excludes file is located without the rest of the gl }), ignore, ); - for (const bad of ['relative/ignore', join(home, 'absent-ignore')]) { + // A relative explicit file refuses even when it exists from this process's working directory. + for (const bad of ['relative/ignore', 'package.json', join(home, 'absent-ignore')]) { await assert.rejects( globalExcludesFile({ repository: '/repository', @@ -474,15 +539,18 @@ test('the candidate copy is byte-exact, keeps only the executable bit and refuse write(join(repository, 'package.json'), '{"name":"root"}\n'); write(join(repository, 'tools', 'run.mjs'), '#!/usr/bin/env node\n', 0o775); write(join(repository, 'docs', 'note.md'), 'note\n', 0o640); + // Group and other execute bits alone do not make a file executable, as in Git's index. + write(join(repository, 'docs', 'group.sh'), 'x\n', 0o654); const destination = fresh('copy'); - const files = ['docs/note.md', 'package.json', 'tools/run.mjs']; - assert.deepEqual(copyCandidateTree({ repository, destination, files }), { files: 3, bytes: 41 }); + const files = ['docs/group.sh', 'docs/note.md', 'package.json', 'tools/run.mjs']; + assert.deepEqual(copyCandidateTree({ repository, destination, files }), { files: 4, bytes: 43 }); for (const path of files) { assert.ok(readFileSync(join(destination, path)).equals(readFileSync(join(repository, path)))); } if (posixModes) { assert.equal(statSync(join(destination, 'tools', 'run.mjs')).mode & 0o777, 0o755); assert.equal(statSync(join(destination, 'docs', 'note.md')).mode & 0o777, 0o644); + assert.equal(statSync(join(destination, 'docs', 'group.sh')).mode & 0o777, 0o644); } if (posixModes) { // The modes are exact whatever the process umask. @@ -570,6 +638,28 @@ test('the dependency store is discovered with store-location variables only', as ]) { assert.equal(key in env, false, key); } + // Every store-location variable reaches the discovery, which runs in the original checkout. + const locations = { + HOME: '/l/home', + USERPROFILE: '/l/profile', + XDG_DATA_HOME: '/l/data', + XDG_CONFIG_HOME: '/l/config', + LOCALAPPDATA: '/l/local', + APPDATA: '/l/appdata', + PNPM_HOME: '/l/pnpm', + npm_config_store_dir: '/l/store-lower', + NPM_CONFIG_STORE_DIR: '/l/store-upper', + }; + await dependencyStore({ + repository: '/repository', + environment: { ...locations, NPM_TOKEN: 'synthetic-secret' }, + home, + pnpm, + }); + assert.equal(calls[1].options.cwd, '/repository'); + for (const [key, value] of Object.entries(locations)) + assert.equal(calls[1].options.env[key], value, key); + assert.equal('NPM_TOKEN' in calls[1].options.env, false); // A caller that already isolated this process names the store explicitly; pnpm is not asked. const explicit = fresh('explicit-store'); assert.equal( @@ -583,7 +673,8 @@ test('the dependency store is discovered with store-location variables only', as }), explicit, ); - for (const bad of ['relative/store', join(scratch, 'absent-store')]) { + // A relative store refuses even when it exists from this process's working directory. + for (const bad of ['relative/store', 'tools', join(scratch, 'absent-store')]) { await assert.rejects( dependencyStore({ repository: '/repository', @@ -594,7 +685,7 @@ test('the dependency store is discovered with store-location variables only', as /DEPENDENCY_STORE/u, ); } - for (const answer of ['relative/store\n', `${join(scratch, 'absent-store')}\n`]) { + for (const answer of ['relative/store\n', 'tools\n', `${join(scratch, 'absent-store')}\n`]) { await assert.rejects( dependencyStore({ repository: '/repository', @@ -610,7 +701,7 @@ test('the dependency store is discovered with store-location variables only', as test('store discovery refuses a private home inside the original checkout', async () => { const repository = fresh('checkout'); // A child whose name merely starts with two dots is inside the checkout too. - for (const home of [join(repository, 'home'), join(repository, '..home')]) { + for (const home of [join(repository, 'home'), join(repository, '..home'), 'relative-home']) { await assert.rejects( dependencyStore({ repository, @@ -741,7 +832,7 @@ function preparation({ let heads = 0; let flags = 0; // With an excludes file, the listing and both status calls answer only when they carry it. - const prefix = excludes === undefined ? '' : `-c core.excludesFile=${excludes} `; + const prefix = READ(excludes); const run = async (command, args) => { const key = args.join(' '); events.push(`git ${key}`); @@ -762,10 +853,12 @@ function preparation({ } throw new Error(`unexpected git ${key}`); }; + let installOptions; const pnpm = async (args, options) => { events.push(`pnpm ${args.join(' ')}`); if (args[0] === 'store') return { stdout: `${store}\n` }; assert.equal(options.cwd, candidate); + installOptions = options; // `misLink` makes the install resolve one engine package, or the root dependency directory, to // the original checkout instead of the candidate. if (misLink === 'root') @@ -779,29 +872,45 @@ function preparation({ } return { stdout: '' }; }; - return { repository, candidate, store, home, events, run, pnpm }; + return { + repository, + candidate, + store, + home, + events, + run, + pnpm, + installOptions: () => installOptions, + }; } test('candidate preparation copies, re-checks identity, installs and verifies links in order', async () => { const ports = preparation(); + const env = { HOME: '/isolated' }; const identity = await prepareCandidate({ repository: ports.repository, candidate: ports.candidate, - environment: { HOME: '/home/someone' }, + environment: { + HOME: '/home/someone', + NPM_TOKEN: 'synthetic-secret', + NODE_OPTIONS: '--require=/x.cjs', + }, home: ports.home, - env: { HOME: '/isolated' }, + env, run: ports.run, pnpm: ports.pnpm, }); + // The install runs in the isolated environment it was given, never the caller's own. + assert.equal(ports.installOptions().env, env); assert.deepEqual(identity, { source: SHA, dirty: false, files: 6 }); assert.deepEqual(ports.events, [ 'git --no-replace-objects rev-parse HEAD', - 'git status --porcelain=v1 -z --untracked-files=all', + `git ${READ()}status --porcelain=v1 -z --untracked-files=all`, 'git ls-files -v -z', - 'git ls-files -z --cached --others --exclude-standard', + `git ${READ()}ls-files -z --cached --others --exclude-standard`, 'git ls-files -z --deleted', 'git --no-replace-objects rev-parse HEAD', - 'git status --porcelain=v1 -z --untracked-files=all', + `git ${READ()}status --porcelain=v1 -z --untracked-files=all`, 'git ls-files -v -z', 'pnpm store path', `pnpm install --offline --frozen-lockfile --ignore-scripts --store-dir ${ports.store}`, @@ -813,6 +922,28 @@ test('candidate preparation copies, re-checks identity, installs and verifies li ); }); +test('a store the caller names reaches the install, and pnpm is never asked to discover one', async () => { + const ports = preparation(); + const explicit = fresh('explicit-store'); + const pnpm = async (args, options) => { + if (args[0] === 'store') throw new Error('store discovery must not run'); + return ports.pnpm(args, options); + }; + await prepareCandidate({ + repository: ports.repository, + candidate: ports.candidate, + environment: { RELAVIUM_PACKAGE_STORE_DIR: explicit }, + home: ports.home, + env: {}, + run: ports.run, + pnpm, + }); + assert.equal( + ports.events.at(-1), + `pnpm install --offline --frozen-lockfile --ignore-scripts --store-dir ${explicit}`, + ); +}); + test('a modified or unchecked source makes the prepared identity dirty', async () => { for (const source of [ { status: ' M package.json\0' }, @@ -895,9 +1026,9 @@ test("candidate preparation applies the developer's global excludes to every lis assert.deepEqual( ports.events.filter((event) => event.includes('status') || event.includes('--cached')), [ - 'git -c core.excludesFile=/developer/ignore status --porcelain=v1 -z --untracked-files=all', - 'git -c core.excludesFile=/developer/ignore ls-files -z --cached --others --exclude-standard', - 'git -c core.excludesFile=/developer/ignore status --porcelain=v1 -z --untracked-files=all', + `git ${READ('/developer/ignore')}status --porcelain=v1 -z --untracked-files=all`, + `git ${READ('/developer/ignore')}ls-files -z --cached --others --exclude-standard`, + `git ${READ('/developer/ignore')}status --porcelain=v1 -z --untracked-files=all`, ], ); }); @@ -919,7 +1050,7 @@ test('an isolated producer receives the store and, when found, the excludes file test('the candidate refuses a file list or a tree over its bounds', async () => { const listing = git({ - 'ls-files -z --cached --others --exclude-standard': 'a.ts\0b.ts\0c.ts\0', + [`${READ()}ls-files -z --cached --others --exclude-standard`]: 'a.ts\0b.ts\0c.ts\0', 'ls-files -z --deleted': '', }); assert.deepEqual(await candidateFiles('/repository', listing.run, undefined, { files: 3 }), [ @@ -965,3 +1096,109 @@ test('the candidate refuses a file list or a tree over its bounds', async () => /FILE_KIND_OR_BOUND/u, ); }); + +test('the production bounds are the documented ones and apply by default', () => { + assert.deepEqual( + { ...CANDIDATE_LIMITS }, + { files: 20_000, fileBytes: 64 * 1024 * 1024, totalBytes: 512 * 1024 * 1024 }, + ); +}); + +test('candidate files and the copy apply the production bounds by default', async () => { + const names = Array.from({ length: 20_001 }, (_, index) => `f${index}.ts`).join('\0'); + const listing = git({ + [`${READ()}ls-files -z --cached --others --exclude-standard`]: names, + 'ls-files -z --deleted': '', + }); + await assert.rejects(candidateFiles('/repository', listing.run), /CANDIDATE_FILES/u); + // A sparse file one byte over the per-file bound is refused from its size, before any read. + const repository = fresh('sparse'); + const descriptor = openSync(join(repository, 'large.bin'), 'w'); + try { + ftruncateSync(descriptor, 64 * 1024 * 1024 + 1); + } finally { + closeSync(descriptor); + } + assert.throws( + () => copyCandidateTree({ repository, destination: fresh('copy'), files: ['large.bin'] }), + /FILE_KIND_OR_BOUND/u, + ); +}); + +test('the copy needs an absolute, empty destination', () => { + const repository = fresh('repository'); + write(join(repository, 'a.ts'), 'a\n'); + assert.throws( + () => + copyCandidateTree({ + repository, + destination: relative(process.cwd(), fresh('copy')), + files: ['a.ts'], + }), + /CANDIDATE_ROOT/u, + ); + const occupied = fresh('copy'); + writeFileSync(join(occupied, 'one'), ''); + assert.throws( + () => copyCandidateTree({ repository, destination: occupied, files: ['a.ts'] }), + /CANDIDATE_ROOT/u, + ); +}); + +test('only the five engine packages may be linked as @relavium workspace packages', () => { + // A link to another workspace package, even one inside the candidate, refuses. + for (const owner of [join('apps', 'cli'), join('packages', 'core')]) { + const root = linkedCandidate(); + mkdirSync(join(root, 'packages', 'ui'), { recursive: true }); + const scope = join(root, ...owner.split('/'), 'node_modules', '@relavium'); + mkdirSync(scope, { recursive: true }); + symlinkSync(join(root, 'packages', 'ui'), join(scope, 'ui'), linkKind); + assert.throws(() => verifyCandidateLinks(root), /CANDIDATE_LINK/u, owner); + } +}); + +test( + 'the excludes lookup never runs a git found in the checkout when the caller has no PATH', + { skip: !posixModes }, + async () => { + // With an empty PATH a spawn would look the command up in its working directory, the checkout. + const repository = fresh('planted-git'); + const marker = join(fresh('marker'), 'ran'); + write(join(repository, 'git'), `#!/bin/sh\necho ran > ${JSON.stringify(marker)}\n`, 0o755); + await globalExcludesFile({ repository, environment: { HOME: fresh('home') } }).catch(() => {}); + assert.equal(existsSync(marker), false); + }, +); + +for (const [label, scope] of [ + [ + 'GIT_CONFIG_COUNT', + (ignore) => ({ + GIT_CONFIG_COUNT: '1', + GIT_CONFIG_KEY_0: 'core.excludesFile', + GIT_CONFIG_VALUE_0: ignore, + }), + ], + [ + 'GIT_CONFIG_PARAMETERS', + (ignore) => ({ GIT_CONFIG_PARAMETERS: `'core.excludesFile'='${ignore}'` }), + ], +]) + test(`a real core.excludesFile set through ${label} is found`, async () => { + const root = fresh('scoped'); + const ignore = join(root, 'scoped-ignore'); + write(ignore, 'secret.local\n'); + mkdirSync(join(root, 'empty-home')); + assert.equal( + await globalExcludesFile({ + repository: fresh('repository'), + environment: { + PATH: process.env.PATH, + HOME: join(root, 'empty-home'), + GIT_CONFIG_NOSYSTEM: '1', + ...scope(ignore), + }, + }), + ignore, + ); + }); diff --git a/tools/cli-package/check.test.mjs b/tools/cli-package/check.test.mjs index 2c6e89be..55e23dbc 100644 --- a/tools/cli-package/check.test.mjs +++ b/tools/cli-package/check.test.mjs @@ -13,7 +13,7 @@ import { writeFileSync, } from 'node:fs'; import { devNull, tmpdir } from 'node:os'; -import { delimiter, dirname, join } from 'node:path'; +import { basename, delimiter, dirname, join } from 'node:path'; import { after, test } from 'node:test'; import { setTimeout as delay } from 'node:timers/promises'; import { fileURLToPath } from 'node:url'; @@ -111,10 +111,11 @@ mkdirSync(process.argv[2]); /** * Start the copied check with stand-ins for its producer and consumer. The store is named - * explicitly, so the check runs no pnpm; so is the excludes file unless `excludes` is false, when - * the check's Git lookup sees an empty home and no global or system configuration. + * explicitly, so the check runs no pnpm, unless a `pnpm` stand-in is given for its store discovery; + * so is the excludes file unless `excludes` is false, when the check's Git lookup sees an empty home + * and no global or system configuration. */ -function check({ pack, consume = CONSUMER, excludes: named = true, git, env: extra = {} }) { +function check({ pack, consume = CONSUMER, excludes: named = true, git, pnpm, env: extra = {} }) { count += 1; const root = join(scratch, `repository-${count}`); for (const path of COPIED) { @@ -134,19 +135,19 @@ function check({ pack, consume = CONSUMER, excludes: named = true, git, env: ext writeFileSync(excludes, ''); const home = join(root, 'home'); mkdirSync(home); - // An optional `git` stand-in, first on PATH, for the check's own excludes lookup. + // Optional `git` and `pnpm` stand-ins, first on PATH, for the check's own excludes lookup and + // store discovery. const bin = join(root, 'bin'); - if (git !== undefined) { - mkdirSync(bin); - writeFileSync(join(bin, 'git'), git, { mode: 0o755 }); - } + mkdirSync(bin); + if (git !== undefined) writeFileSync(join(bin, 'git'), git, { mode: 0o755 }); + if (pnpm !== undefined) writeFileSync(join(bin, 'pnpm'), pnpm(markers, store), { mode: 0o755 }); writeFileSync(join(root, 'tools', 'cli-package', 'pack.mjs'), pack(markers)); writeFileSync(join(root, 'tools', 'cli-package', 'consume.mjs'), consume(markers)); const child = spawn(process.execPath, [join(root, 'tools', 'cli-package', 'check.mjs')], { env: { - PATH: git === undefined ? process.env.PATH : `${bin}${delimiter}${process.env.PATH}`, + PATH: `${bin}${delimiter}${process.env.PATH}`, TMPDIR: temporary, - RELAVIUM_PACKAGE_STORE_DIR: store, + ...(pnpm === undefined ? { RELAVIUM_PACKAGE_STORE_DIR: store } : {}), ...(named ? { RELAVIUM_GIT_EXCLUDES_FILE: excludes } : { HOME: home, GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: devNull }), @@ -155,11 +156,13 @@ function check({ pack, consume = CONSUMER, excludes: named = true, git, env: ext stdio: ['ignore', 'pipe', 'pipe'], }); let stdout = ''; + let stderr = ''; child.stdout.on('data', (chunk) => (stdout += chunk)); + child.stderr.on('data', (chunk) => (stderr += chunk)); const exited = new Promise((resolve) => - child.once('exit', (code, signal) => resolve({ code, signal, stdout })), + child.once('exit', (code, signal) => resolve({ code, signal, stdout, stderr })), ); - return { child, exited, markers, temporary, store, excludes }; + return { child, exited, root, markers, temporary, store, excludes }; } /** The check's exit, bounded so that a regressed check fails its test instead of hanging it. */ @@ -188,6 +191,14 @@ function alive(pid) { } } +/** A killed process can still answer for a moment after its killer exits: wait, with a bound. */ +async function assertGone(pid, message) { + for (let waited = 0; alive(pid); waited += 25) { + assert.ok(waited < 5_000, message); + await delay(25); + } +} + /** * Stop the check and every stand-in a regressed check could leave behind: the stand-ins run in their * own process groups, so they would outlive a check that died without forwarding its interrupt. @@ -227,7 +238,7 @@ test( assert.equal(outcome.code, 130); assert.equal(outcome.stdout, ''); // The consumer received the forwarded interrupt and stopped its writer before its home went. - assert.equal(alive(pid), false); + await assertGone(pid, 'the consumer writer outlived the interrupt'); assert.deepEqual(checkRoots(run.temporary), []); } finally { stopAll(run); @@ -318,6 +329,43 @@ test('a successful producer and consumer give the summary and leave no check roo } }); +test('without a named store the check asks pnpm in the repository, with its own private home', async () => { + // The private home must lie outside the repository, so the temporary directory does too. + const outside = mkdtempSync(join(scratch, 'outside-')); + const developer = join(outside, 'developer-home'); + const run = check({ + env: { TMPDIR: outside, HOME: developer }, + pack: RECORDING_PRODUCER, + consume: PUBLISHING_CONSUMER('local'), + pnpm: (markers, store) => + '#!/usr/bin/env node\n' + + `require('node:fs').writeFileSync(${JSON.stringify(join(markers, 'pnpm-call'))}, ` + + 'JSON.stringify({ args: process.argv.slice(2), cwd: process.cwd(), ' + + 'home: process.env.HOME, temporary: process.env.TMPDIR, cache: process.env.XDG_CACHE_HOME }));\n' + + `process.stdout.write(${JSON.stringify(`${store}\n`)});\n`, + }); + try { + const outcome = await exitOf(run); + assert.equal(outcome.code, 0, outcome.stderr); + const call = JSON.parse(readFileSync(join(run.markers, 'pnpm-call'), 'utf8')); + assert.deepEqual(call.args, ['store', 'path']); + assert.equal(realpathSync(call.cwd), realpathSync(run.root)); + // The store location follows the developer's own HOME; temporary files and caches go to the + // check's owner-only root. + assert.equal(call.home, developer); + assert.equal(dirname(call.temporary), outside); + assert.match(basename(call.temporary), /^relavium-local-package-check-/u); + assert.equal(call.cache, call.temporary); + assert.equal( + JSON.parse(readFileSync(join(run.markers, 'pack-environment'), 'utf8')).store, + run.store, + ); + assert.deepEqual(checkRoots(outside), []); + } finally { + stopAll(run); + } +}); + test('without a global excludes file the producer receives none', async () => { const run = check({ pack: RECORDING_PRODUCER, @@ -443,7 +491,7 @@ await exitOnInterrupt( assert.equal(outcome.code, 1); assert.equal(outcome.stdout, ''); assert.equal(readFileSync(join(run.markers, 'pack-interrupted'), 'utf8'), 'cleaned'); - assert.equal(alive(grandchild), false); + await assertGone(grandchild, 'the stage build child outlived the deadline'); // Past the moment the build child would have written: nothing recreated the check root. await delay(3500); assert.equal(existsSync(join(run.temporary, checkRoot)), false); @@ -553,7 +601,7 @@ await completeConsumer({ assert.equal(outcome.code, 1); assert.equal(outcome.stdout, ''); assert.equal(readFileSync(join(run.markers, 'consume-cleaned'), 'utf8'), 'cleaned'); - assert.equal(alive(grandchild), false); + await assertGone(grandchild, 'the stage build child outlived the deadline'); await delay(3500); assert.equal(existsSync(join(run.temporary, checkRoot)), false); } finally { diff --git a/tools/cli-package/child-exit.mjs b/tools/cli-package/child-exit.mjs new file mode 100644 index 00000000..50df296a --- /dev/null +++ b/tools/cli-package/child-exit.mjs @@ -0,0 +1,24 @@ +/** + * Test-only: a spawned tool's exit within a bound, so that a regressed tool fails its test instead of + * hanging it, which would let the runner kill the test and leave the tool's detached descendants + * behind. Call it before signalling the child. + */ +import assert from 'node:assert/strict'; +import { setTimeout as delay } from 'node:timers/promises'; + +/** + * @param {import('node:child_process').ChildProcess} child + * @param {number} [timeoutMs] + * @returns {Promise<[number | null, string | null]>} the exit code and signal + */ +export function exitWithin(child, timeoutMs = 30_000) { + const exit = new Promise((resolve) => + child.once('exit', (code, signal) => resolve([code, signal])), + ); + // Unreferenced, so a pending bound never keeps the test process alive after the child exits. + const bound = delay(timeoutMs, undefined, { ref: false }); + return Promise.race([exit, bound]).then((outcome) => { + assert.ok(outcome !== undefined, `the child did not exit within ${timeoutMs} ms`); + return outcome; + }); +} diff --git a/tools/cli-package/consume.mjs b/tools/cli-package/consume.mjs index bd42a21f..a76cb822 100644 --- a/tools/cli-package/consume.mjs +++ b/tools/cli-package/consume.mjs @@ -12,7 +12,7 @@ import { writeSync, } from 'node:fs'; import { tmpdir } from 'node:os'; -import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; +import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { runBootstrap } from '../cli-bootstrap/check.mjs'; import { inspectPackageArchive } from './archive.mjs'; @@ -22,6 +22,7 @@ import { validateConsumerRecord, } from './binding.mjs'; import { installedInventory, oneTarball, readOrdinaryFile, sha256 } from './files.mjs'; +import { placedWithin, within } from './paths.mjs'; import { exerciseInstalledCommand, isolatedEnvironment, @@ -64,11 +65,8 @@ inspectPackageArchive(bytes, { expectedPackage: receipt.package, }); const home = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-package-consumer-'))); -const inside = (parent, child) => { - const part = relative(parent, child); - return part === '' || (!isAbsolute(part) && part !== '..' && !part.startsWith(`..${sep}`)); -}; -assert.equal(inside(realpathSync(repository), home), false, 'PACKED_CLI_INSTALL_IN_WORKSPACE'); +// Compared on the stored letter case, so a TMPDIR naming the checkout in another case is refused. +assert.equal(placedWithin(repository, home), false, 'PACKED_CLI_INSTALL_IN_WORKSPACE'); await completeConsumer({ async exercise() { const env = isolatedEnvironment(home); @@ -99,7 +97,7 @@ await completeConsumer({ ); writeFileSync(join(reportDirectory, 'install.log'), installation.stdout + installation.stderr); const packageRoot = realpathSync(join(home, 'node_modules/relavium')); - assert.ok(inside(home, packageRoot), 'PACKED_CLI_INSTALL_ESCAPED'); + assert.ok(within(home, packageRoot), 'PACKED_CLI_INSTALL_ESCAPED'); const before = installedInventory(packageRoot); const wanted = receipt.files .map(({ path, bytes, sha256 }) => ({ path, bytes, sha256 })) @@ -123,10 +121,10 @@ await completeConsumer({ name === '@modelcontextprotocol/sdk' ? '@modelcontextprotocol/sdk/client/index.js' : name, ), ); - assert.ok(inside(home, entry), 'PACKED_CLI_DEPENDENCY_OUTSIDE_INSTALL'); + assert.ok(within(home, entry), 'PACKED_CLI_DEPENDENCY_OUTSIDE_INSTALL'); let directory = dirname(entry), found = false; - while (inside(home, directory)) { + while (within(home, directory)) { let content; try { content = readOrdinaryFile(join(directory, 'package.json'), 1024 * 1024); diff --git a/tools/cli-package/consumer-lifetime.test.mjs b/tools/cli-package/consumer-lifetime.test.mjs index 792b1a74..91f9ffb0 100644 --- a/tools/cli-package/consumer-lifetime.test.mjs +++ b/tools/cli-package/consumer-lifetime.test.mjs @@ -18,8 +18,10 @@ import { dirname, join } from 'node:path'; import { after, test } from 'node:test'; import { setTimeout as delay } from 'node:timers/promises'; import { pathToFileURL } from 'node:url'; +import { exitWithin } from './child-exit.mjs'; import { completeConsumer } from './consumer-lifetime.mjs'; import { REQUIRED_SMOKE_CASES } from './smoke/cases.mjs'; +import { abandonInstalledSmoke, createInstalledSmokeRoot } from './smoke/index.mjs'; // A support directory for the cases that never reach an interrupt. const synthetic = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-consumer-support-'))); @@ -382,9 +384,7 @@ await completeConsumer({ const children = []; try { for (const file of pids) children.push(await waitForPid(file)); - const exited = new Promise((resolve) => - consumer.once('exit', (code, signal) => resolve([code, signal])), - ); + const exited = exitWithin(consumer); consumer.kill('SIGINT'); const [code, signal] = await exited; for (let waited = 0; children.some(alive); waited += 25) { @@ -432,3 +432,66 @@ test( assert.equal(outcome.events, 'cleanup fixture:gone smoke:gone\ncleanupFailed\n'); }, ); + +test( + 'an interrupt still removes the installation when an earlier abandonment step fails', + { skip: process.platform === 'win32' }, + async (t) => { + const root = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-consumer-steps-'))); + t.after(() => rmSync(root, { recursive: true, force: true })); + const installation = join(root, 'installation'); + mkdirSync(installation); + const log = join(root, 'events'); + const pidFile = join(root, 'child.pid'); + const url = (name) => JSON.stringify(new URL(name, import.meta.url).href); + // An absolute support directory that does not exist, so abandoning the smoke roots throws. + const script = ` +import { appendFileSync, rmSync } from 'node:fs'; +import { completeConsumer } from ${url('./consumer-lifetime.mjs')}; +import { isolatedEnvironment, runCommand } from ${url('./process.mjs')}; +const record = (event) => appendFileSync(${JSON.stringify(log)}, event + '\\n'); +await completeConsumer({ + supportRoot: ${JSON.stringify(join(root, 'missing-support'))}, + exercise: () => + runCommand(process.execPath, ['-e', ${JSON.stringify(writer(JSON.stringify(installation), pidFile))}], { + env: isolatedEnvironment(${JSON.stringify(root)}), + timeoutMs: 60_000, + }), + cleanup: () => { + rmSync(${JSON.stringify(installation)}, { recursive: true, force: true, maxRetries: 3 }); + record('cleanup'); + }, + cleanupFailed: () => record('cleanupFailed'), + publish: () => record('publish'), +}); +`; + const consumer = spawn(process.execPath, ['--input-type=module', '-e', script], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + let child; + try { + child = await waitForPid(pidFile); + const exited = exitWithin(consumer); + consumer.kill('SIGINT'); + assert.deepEqual(await exited, [130, null]); + assert.equal(existsSync(installation), false); + assert.equal(readFileSync(log, 'utf8'), 'cleanup\ncleanupFailed\n'); + } finally { + if (consumer.exitCode === null && consumer.signalCode === null) consumer.kill('SIGKILL'); + if (child !== undefined && alive(child)) process.kill(child, 'SIGKILL'); + } + }, +); + +test('abandoning the installed smoke removes exactly the roots the worker creates', (t) => { + const support = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-smoke-roots-'))); + t.after(() => rmSync(support, { recursive: true, force: true })); + const roots = [createInstalledSmokeRoot(support), createInstalledSmokeRoot(support)]; + for (const root of roots) { + assert.equal(dirname(root), support); + writeFileSync(join(root, 'guard.ndjson'), '{}\n'); + } + writeFileSync(join(support, 'installed-smoke.json'), '{}\n'); + abandonInstalledSmoke(support); + assert.deepEqual(readdirSync(support), ['installed-smoke.json']); +}); diff --git a/tools/cli-package/finalize.mjs b/tools/cli-package/finalize.mjs index 9758085e..291b8622 100644 --- a/tools/cli-package/finalize.mjs +++ b/tools/cli-package/finalize.mjs @@ -6,9 +6,11 @@ * stages the maps and the build metadata owner-only outside the package, re-checks the finite * closure and the public file/mode inventory, and only then hashes. Nothing writes JS afterwards. * - * Refusals name a fixed code and at most a repository-relative output or source path, never map or - * source contents. A successful bundle is no evidence by itself: a missing upstream map makes the - * bundler silently map engine frames to compiled JS, so every source set is checked explicitly. + * Refusals name a fixed code and at most a fixed label or a repository-relative path that has passed + * validation, never map or source contents and never a host path: a metafile string that escapes + * the repository or is not a plain path is left out. A successful bundle is no evidence by itself: a + * missing upstream map makes the bundler silently map engine frames to compiled JS, so every source + * set is checked explicitly. */ import { Buffer } from 'node:buffer'; import { @@ -51,9 +53,10 @@ const LIMITS = Object.freeze({ }); const SOURCE_TAIL = /^(?:[A-Za-z0-9_-][A-Za-z0-9_.-]*\/)*[A-Za-z0-9_-][A-Za-z0-9_.-]*\.tsx?$/u; const MAPPINGS = /^[A-Za-z0-9+/,;]*$/u; +const PLAIN_PATH = /^(?:[A-Za-z0-9_-][A-Za-z0-9_.-]*\/)*[A-Za-z0-9_-][A-Za-z0-9_.-]*$/u; export class CandidateFinalizationError extends Error { - /** @param {string} code @param {string} [subject] a repository-relative path, never contents */ + /** @param {string} code @param {string} [subject] a fixed label or a validated repository-relative path */ constructor(code, subject) { super( `CLI candidate finalization refused: ${code}` + @@ -213,6 +216,17 @@ function upstreamSources(repository, packageName, jsPath) { ); } +/** + * A metafile input (relative to apps/cli) as a refusal subject: its repository-relative path when + * that is a plain path inside the repository, otherwise nothing. + * @param {string} input + */ +function inputSubject(input) { + if (input.startsWith('/')) return undefined; + const path = posix.normalize(posix.join('apps/cli', input)); + return path.length <= 200 && PLAIN_PATH.test(path) ? path : undefined; +} + /** * Original TypeScript sources behind one metafile input: the CLI's own source directly, an engine * package's compiled JS through its tsc map. Any other input kind refuses. @@ -225,7 +239,9 @@ function inputSources(repository, input, cache) { let sources; const engine = /^\.\.\/\.\.\/packages\/([a-z]+)\/dist\/(.+\.js)$/u.exec(input); if (input.startsWith('src/') && SOURCE_TAIL.test(input.slice(4)) && !input.endsWith('.d.ts')) { - sources = [originalSource(repository, 'apps/cli', input, ['apps/cli/src'], input)]; + sources = [ + originalSource(repository, 'apps/cli', input, ['apps/cli/src'], `apps/cli/${input}`), + ]; } else if ( engine && ENGINE_PACKAGES.includes(engine[1]) && @@ -233,7 +249,7 @@ function inputSources(repository, input, cache) { ) { sources = upstreamSources(repository, engine[1], `packages/${engine[1]}/dist/${engine[2]}`); } else { - refuse('UNSUPPORTED_INPUT', input); + refuse('UNSUPPORTED_INPUT', inputSubject(input)); } cache.set(input, sources); return sources; @@ -381,7 +397,7 @@ export function finalizeCandidate({ repository, staging }) { if (runtime.length === 0 || runtime.length > LIMITS.outputs) refuse('OUTPUT_INVENTORY'); const expectedEntries = ['metafile-esm.json']; for (const path of runtime) { - if (!/^dist\/[A-Za-z0-9_-]+\.js$/u.test(path)) refuse('OUTPUT_INVENTORY', `apps/cli/${path}`); + if (!/^dist\/[A-Za-z0-9_-]+\.js$/u.test(path)) refuse('OUTPUT_INVENTORY'); if (!Object.hasOwn(metafile.outputs, `${path}.map`)) refuse('MAP_PAIRING', `apps/cli/${path}`); expectedEntries.push(basename(path), `${basename(path)}.map`); } @@ -419,7 +435,7 @@ export function finalizeCandidate({ repository, staging }) { } if (contribution.bytesInOutput === 0) continue; const sources = cache.get(input); - if (!sources) refuse('UNSUPPORTED_INPUT', input); + if (!sources) refuse('UNSUPPORTED_INPUT', inputSubject(input)); for (const source of sources) expected.add(source); } const map = finalMap( diff --git a/tools/cli-package/finalize.test.mjs b/tools/cli-package/finalize.test.mjs index 1364e45e..d375ee13 100644 --- a/tools/cli-package/finalize.test.mjs +++ b/tools/cli-package/finalize.test.mjs @@ -1,6 +1,10 @@ import assert from 'node:assert/strict'; +import { Buffer } from 'node:buffer'; +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; import { chmodSync, + copyFileSync, existsSync, mkdirSync, mkdtempSync, @@ -16,7 +20,7 @@ import { createRequire } from 'node:module'; import { tmpdir } from 'node:os'; import { basename, dirname, join, relative, sep } from 'node:path'; import { after, test } from 'node:test'; -import { fileURLToPath } from 'node:url'; +import { fileURLToPath, pathToFileURL } from 'node:url'; import ts from 'typescript'; import { BundleClosureError } from '../bundle-closure/closure.mjs'; import { CandidateFinalizationError, finalizeCandidate, mapTrailer } from './finalize.mjs'; @@ -25,7 +29,7 @@ import { CandidateFinalizationError, finalizeCandidate, mapTrailer } from './fin // upstream and final maps from tiny fixtures; tsup's writer is reproduced exactly (trailer appended // to the chunk code, map JSON re-serialized beside it). No product build runs here. const cliRequire = createRequire(new URL('../../apps/cli/package.json', import.meta.url)); -const { build } = createRequire(cliRequire.resolve('tsup/package.json'))('esbuild'); +const { build, transformSync } = createRequire(cliRequire.resolve('tsup/package.json'))('esbuild'); const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-candidate-finalizer-'))); after(() => rmSync(scratch, { recursive: true, force: true })); const posixModes = process.platform !== 'win32'; @@ -54,7 +58,7 @@ function snapshot(directory) { * facade: a statically and dynamically imported module (no inputs) and a pure re-export module * (a zero-byte input). */ -async function fixture({ upstreamMapAtBundle = true, sourcesContent = false } = {}) { +async function fixture({ upstreamMapAtBundle = true, sourcesContent = false, tsx = false } = {}) { count += 1; const root = join(scratch, `repository-${count}`); const core = join(root, 'packages', 'core'); @@ -101,6 +105,14 @@ async function fixture({ upstreamMapAtBundle = true, sourcesContent = false } = join(cli, 'src', 'facade.ts'), "export { greet } from '../../../packages/core/dist/index.js';\n", ); + if (tsx) { + write(join(cli, 'src', 'view.tsx'), "export const view = (): string => 'view';\n"); + write( + join(cli, 'src', 'index.ts'), + readFileSync(join(cli, 'src', 'index.ts'), 'utf8') + + "import { view } from './view.js';\nconsole.log(view());\n", + ); + } const built = await build({ absWorkingDir: cli, entryPoints: { index: 'src/index.ts' }, @@ -150,6 +162,18 @@ function refusesBeforeWriting(candidate, code) { assert.deepEqual(readdirSync(candidate.staging), []); } +/** The finalizer's refusal of `candidate`; accepting it fails the test. */ +function refusal(candidate) { + try { + finalizeCandidate({ repository: candidate.root, staging: candidate.staging }); + } catch (error) { + return error; + } + assert.fail('the finalizer accepted the candidate'); +} + +const digest = (bytes) => createHash('sha256').update(bytes).digest('hex'); + function editJson(path, change) { const value = JSON.parse(readFileSync(path, 'utf8')); change(value); @@ -213,9 +237,23 @@ test('finalizes a chained candidate into private repository-relative maps and pu JSON.parse(readFileSync(join(candidate.staging, 'finalized.json'), 'utf8')), record, ); + // The private record hashes exactly the finalized JS and the staged maps. + assert.equal(record.version, 1); + assert.deepEqual( + record.outputs.map((output) => output.path).sort(), + names.map((name) => `dist/${name}`), + ); for (const output of record.outputs) { const bytes = readFileSync(join(candidate.cli, output.path)); - assert.equal(output.bytes, bytes.length); + assert.deepEqual(output, { path: output.path, bytes: bytes.length, sha256: digest(bytes) }); + } + assert.deepEqual( + record.maps.map((map) => map.path).sort(), + names.map((name) => `dist/${name}.map`), + ); + for (const map of record.maps) { + const bytes = readFileSync(join(candidate.staging, 'maps', basename(map.path))); + assert.deepEqual(map, { path: map.path, bytes: bytes.length, sha256: digest(bytes) }); } const facades = facadeNames(candidate); assert.equal(facades.length, 2, 'both facade kinds are exercised'); @@ -407,13 +445,22 @@ test('accounts for every emitted file and every consumed input', async () => { delete metafile.outputs['dist/index.js.map']; }); refusesBeforeWriting(unpaired, 'MAP_PAIRING'); - const crowded = await fixture(); - editJson(join(crowded.dist, 'metafile-esm.json'), (metafile) => { - for (let i = 0; i < 257; i += 1) { - metafile.outputs[`dist/extra${i}.js`] = { imports: [], exports: [], inputs: {}, bytes: 1 }; - } - }); - refusesBeforeWriting(crowded, 'OUTPUT_INVENTORY'); + // Exactly 256 runtime outputs pass the bound (and then lack the last one's map entry); 257 refuse. + for (const [total, code] of [ + [257, 'OUTPUT_INVENTORY'], + [256, 'MAP_PAIRING'], + ]) { + const crowded = await fixture(); + editJson(join(crowded.dist, 'metafile-esm.json'), (metafile) => { + const runtime = Object.keys(metafile.outputs).filter((key) => !key.endsWith('.map')); + const extra = { imports: [], exports: [], inputs: {}, bytes: 1 }; + for (let i = 0; i < total - runtime.length; i += 1) { + metafile.outputs[`dist/extra${i}.js`] = extra; + if (i < total - runtime.length - 1) metafile.outputs[`dist/extra${i}.js.map`] = extra; + } + }); + refusesBeforeWriting(crowded, code); + } const unsized = await fixture(); editJson(join(unsized.dist, 'metafile-esm.json'), (metafile) => { const contribution = Object.values(metafile.outputs['dist/index.js'].inputs)[0]; @@ -429,14 +476,36 @@ test('accounts for every emitted file and every consumed input', async () => { }; }); refusesBeforeWriting(foreign, 'UNSUPPORTED_INPUT'); - // Only the five inlined engine packages and the CLI's own TypeScript are supported inputs. - for (const input of ['../../packages/ui/dist/x.js', 'src/types.d.ts']) { + // Only the five inlined engine packages' canonical compiled JS and the CLI's own TypeScript are + // supported inputs, even where a non-canonical spelling names a real compiled file. + for (const input of [ + '../../packages/ui/dist/x.js', + 'src/types.d.ts', + 'src/x.js', + '../../packages/core/dist/../dist/index.js', + 'src/../../../packages/core/dist/index.js', + 'x/../../packages/core/dist/index.js', + ]) { const unsupported = await fixture(); + write(join(unsupported.cli, 'src', 'x.js'), 'export {};\n'); editJson(join(unsupported.dist, 'metafile-esm.json'), (metafile) => { metafile.inputs[input] = { bytes: 1, imports: [], format: 'esm' }; }); refusesBeforeWriting(unsupported, 'UNSUPPORTED_INPUT'); } + // An input contributing bytes to an output must be one of the metafile's inputs. + const ghost = await fixture(); + editJson(join(ghost.dist, 'metafile-esm.json'), (metafile) => { + metafile.outputs['dist/index.js'].inputs['src/ghost.ts'] = { bytesInOutput: 1 }; + }); + refusesBeforeWriting(ghost, 'UNSUPPORTED_INPUT'); + // A build with no runtime output refuses before anything is written. + const empty = await fixture(); + for (const name of readdirSync(empty.dist)) { + if (name !== 'metafile-esm.json') rmSync(join(empty.dist, name)); + } + editJson(join(empty.dist, 'metafile-esm.json'), (metafile) => (metafile.outputs = {})); + refusesBeforeWriting(empty, 'OUTPUT_INVENTORY'); if (posixModes) { const linked = await fixture(); symlinkSync(join(linked.dist, 'index.js'), join(linked.dist, 'alias.js')); @@ -498,28 +567,276 @@ test('a source root is a whole directory: a sibling that only shares its prefix refusesBeforeWriting(final, 'MAP_SOURCE_NOT_ORIGINAL'); }); -test('refuses a metafile without outputs or inputs', async () => { - for (const field of ['outputs', 'inputs']) { +test('refuses a metafile whose records have the wrong type', async () => { + for (const change of [ + (metafile) => delete metafile.outputs, + (metafile) => delete metafile.inputs, + (metafile) => (metafile.outputs = []), + (metafile) => (metafile.inputs = []), + (metafile) => (metafile.outputs['dist/index.js'] = null), + (metafile) => delete metafile.outputs['dist/index.js'].inputs, + (metafile) => (metafile.outputs['dist/index.js'].inputs = []), + (metafile) => (metafile.outputs['dist/index.js'].bytes = '12'), + (metafile) => { + const inputs = metafile.outputs['dist/index.js'].inputs; + inputs[Object.keys(inputs)[0]] = null; + }, + ]) { const candidate = await fixture(); - editJson(join(candidate.dist, 'metafile-esm.json'), (metafile) => delete metafile[field]); + editJson(join(candidate.dist, 'metafile-esm.json'), change); refusesBeforeWriting(candidate, 'METAFILE_SHAPE'); } + const absent = await fixture(); + writeFileSync(join(absent.dist, 'metafile-esm.json'), 'null'); + refusesBeforeWriting(absent, 'METAFILE_SHAPE'); }); -test('refuses a metafile, an upstream map or a final map over its byte bound', async () => { - // Valid JSON padded past each bound, so only the bound refuses it. +test('reads a metafile, an upstream map or a final map of exactly its byte bound, and no more', async () => { + // Valid JSON padded to each bound, so only the bound decides. for (const [path, bound] of [ [(candidate) => join(candidate.dist, 'metafile-esm.json'), 4 * 1024 * 1024], [(candidate) => join(candidate.core, 'dist', 'index.js.map'), 4 * 1024 * 1024], [(candidate) => join(candidate.dist, 'index.js.map'), 16 * 1024 * 1024], + ]) { + for (const extra of [0, 1]) { + const candidate = await fixture(); + const file = path(candidate); + const text = readFileSync(file, 'utf8'); + writeFileSync(file, text + ' '.repeat(bound - Buffer.byteLength(text) + extra)); + if (extra === 0) + finalizeCandidate({ repository: candidate.root, staging: candidate.staging }); + else refusesBeforeWriting(candidate, 'UNREADABLE_FILE'); + } + } + // Final and upstream JS over the JS bound. + for (const path of [ + (candidate) => join(candidate.dist, 'index.js'), + (candidate) => join(candidate.core, 'dist', 'index.js'), ]) { const candidate = await fixture(); const file = path(candidate); - writeFileSync(file, readFileSync(file, 'utf8') + ' '.repeat(bound)); + writeFileSync(file, ' '.repeat(8 * 1024 * 1024) + readFileSync(file, 'utf8')); refusesBeforeWriting(candidate, 'UNREADABLE_FILE'); } }); +test('decodes JS and maps strictly: no byte-order mark is removed or admitted, no invalid UTF-8', async () => { + const bom = Buffer.from([0xef, 0xbb, 0xbf]); + // The leading mark would change the public bytes, so the recorded size no longer matches. + const js = await fixture(); + writeFileSync( + join(js.dist, 'index.js'), + Buffer.concat([bom, readFileSync(join(js.dist, 'index.js'))]), + ); + refusesBeforeWriting(js, 'TRAILER_BYTES'); + const map = await fixture(); + writeFileSync( + join(map.dist, 'index.js.map'), + Buffer.concat([bom, readFileSync(join(map.dist, 'index.js.map'))]), + ); + refusesBeforeWriting(map, 'MALFORMED_JSON'); + const invalidMap = await fixture(); + const text = readFileSync(join(invalidMap.dist, 'index.js.map'), 'utf8'); + writeFileSync( + join(invalidMap.dist, 'index.js.map'), + Buffer.concat([Buffer.from(`${text.slice(0, -1)},"file":"`), Buffer.from([0xff, 0x22, 0x7d])]), + ); + refusesBeforeWriting(invalidMap, 'MALFORMED_JSON'); + const invalidUpstream = await fixture(); + const upstream = join(invalidUpstream.core, 'dist', 'index.js'); + writeFileSync( + upstream, + Buffer.concat([Buffer.from([0x2f, 0x2f, 0xff, 0x0a]), readFileSync(upstream)]), + ); + refusesBeforeWriting(invalidUpstream, 'UNREADABLE_FILE'); +}); + +test('refuses upstream and final maps whose fields have the wrong type', async () => { + for (const change of [ + (map) => (map.mappings = 1234), + (map) => (map.names = 'x'), + (map) => (map.names = {}), + (map) => (map.sources = 'x'), + (map) => (map.sources = {}), + ]) { + const upstream = await fixture(); + editJson(join(upstream.core, 'dist', 'index.js.map'), change); + refusesBeforeWriting(upstream, 'UPSTREAM_MAP_SHAPE'); + const final = await fixture(); + editJson(join(final.dist, 'index.js.map'), change); + refusesBeforeWriting(final, 'MAP_SHAPE'); + } + const upstream = await fixture(); + writeFileSync(join(upstream.core, 'dist', 'index.js.map'), 'null'); + refusesBeforeWriting(upstream, 'UPSTREAM_MAP_SHAPE'); + const final = await fixture(); + writeFileSync(join(final.dist, 'index.js.map'), 'null'); + refusesBeforeWriting(final, 'MAP_SHAPE'); + if (posixModes) { + // A link standing in for an upstream map is not one. + const linked = await fixture(); + const map = join(linked.core, 'dist', 'index.js.map'); + writeFileSync(join(linked.root, 'elsewhere.map'), readFileSync(map)); + rmSync(map); + symlinkSync(join(linked.root, 'elsewhere.map'), map); + refusesBeforeWriting(linked, 'UPSTREAM_MAP_MISSING'); + } +}); + +test('a map source must be a plain path of at most 1024 characters to an original file', async () => { + for (const source of [ + '../src/index%2Ets', + '../src/index.ts?x', + '../src/index.ts#x', + '../src/in\u0001dex.ts', + '', + `../src/${'a/'.repeat(510)}x.ts`, + 1, + null, + ['../src/index.ts'], + ]) { + const upstream = await fixture(); + editJson(join(upstream.core, 'dist', 'index.js.map'), (map) => (map.sources = [source])); + refusesBeforeWriting(upstream, 'MAP_SOURCE_PATH'); + if (typeof source !== 'string') { + const final = await fixture(); + editJson(join(final.dist, 'index.js.map'), (map) => (map.sources[0] = source)); + refusesBeforeWriting(final, 'MAP_SOURCE_PATH'); + } + } + // A source of exactly 1024 characters passes the path check and is then looked up. + const bounded = await fixture(); + const longest = `../src/${'a/'.repeat(506)}xy.ts`; + assert.equal(longest.length, 1024); + editJson(join(bounded.core, 'dist', 'index.js.map'), (map) => (map.sources = [longest])); + refusesBeforeWriting(bounded, 'MAP_SOURCE_MISSING'); + // A source under a dot directory is not original TypeScript, even when the file exists. + const dotted = await fixture(); + write(join(dotted.core, 'src', '.hidden', 'x.ts'), 'export const x = 1;\n'); + editJson( + join(dotted.core, 'dist', 'index.js.map'), + (map) => (map.sources = ['../src/.hidden/x.ts']), + ); + refusesBeforeWriting(dotted, 'MAP_SOURCE_NOT_ORIGINAL'); + if (posixModes) { + // A link to TypeScript elsewhere is not an original source file. + const linked = await fixture(); + writeFileSync(join(linked.root, 'real.ts'), 'export const x = 1;\n'); + symlinkSync(join(linked.root, 'real.ts'), join(linked.core, 'src', 'linked.ts')); + editJson( + join(linked.core, 'dist', 'index.js.map'), + (map) => (map.sources = ['../src/linked.ts']), + ); + refusesBeforeWriting(linked, 'MAP_SOURCE_MISSING'); + } +}); + +test('admits original .tsx CLI sources', async () => { + const candidate = await fixture({ tsx: true }); + finalizeCandidate({ repository: candidate.root, staging: candidate.staging }); + const map = JSON.parse(readFileSync(join(candidate.staging, 'maps', 'index.js.map'), 'utf8')); + assert.ok(map.sources.includes('apps/cli/src/view.tsx'), JSON.stringify(map.sources)); +}); + +test('a refusal names at most a validated repository-relative subject, never a host path', async () => { + const input = (path) => (candidate) => + editJson(join(candidate.dist, 'metafile-esm.json'), (metafile) => { + metafile.inputs[typeof path === 'function' ? path(candidate) : path] = { + bytes: 1, + imports: [], + format: 'esm', + }; + }); + const upstreamMap = (candidate) => join(candidate.core, 'dist', 'index.js.map'); + const finalMap = (candidate) => join(candidate.dist, 'index.js.map'); + for (const [change, code, subject] of [ + [(c) => rmSync(upstreamMap(c)), 'UPSTREAM_MAP_MISSING', 'packages/core/dist/index.js'], + [ + (c) => editJson(upstreamMap(c), (map) => (map.version = 2)), + 'UPSTREAM_MAP_SHAPE', + 'packages/core/dist/index.js.map', + ], + [ + (c) => editJson(upstreamMap(c), (map) => (map.sources = ['../src/missing.ts'])), + 'MAP_SOURCE_MISSING', + 'packages/core/dist/index.js.map', + ], + [ + (c) => editJson(upstreamMap(c), (map) => (map.sources = [`${c.root}/x.ts`])), + 'MAP_SOURCE_PATH', + 'packages/core/dist/index.js.map', + ], + [ + (c) => writeFileSync(upstreamMap(c), '{'), + 'MALFORMED_JSON', + 'packages/core/dist/index.js.map', + ], + [ + (c) => writeFileSync(upstreamMap(c), ' '.repeat(4 * 1024 * 1024 + 1)), + 'UNREADABLE_FILE', + 'packages/core/dist/index.js.map', + ], + [ + (c) => rmSync(join(c.core, 'dist', 'types.js')), + 'UNREADABLE_FILE', + 'packages/core/dist/types.js', + ], + [ + (c) => editJson(finalMap(c), (map) => (map.version = 2)), + 'MAP_SHAPE', + 'apps/cli/dist/index.js.map', + ], + [ + (c) => writeFileSync(finalMap(c), ' '.repeat(16 * 1024 * 1024 + 1)), + 'UNREADABLE_FILE', + 'apps/cli/dist/index.js.map', + ], + [ + (c) => writeFileSync(join(c.dist, 'index.js'), `${c.emitted['index.js']}\n`), + 'FINAL_MAP_LINK', + 'apps/cli/dist/index.js', + ], + [(c) => writeFileSync(join(c.dist, 'metafile-esm.json'), '{'), 'MALFORMED_JSON', 'metafile'], + [input('src/missing.ts'), 'MAP_SOURCE_MISSING', 'apps/cli/src/missing.ts'], + [input('../../node_modules/x/index.js'), 'UNSUPPORTED_INPUT', 'node_modules/x/index.js'], + [ + (c) => + editJson(join(c.dist, 'metafile-esm.json'), (metafile) => { + metafile.outputs['dist/index.js'].inputs['src/ghost.ts'] = { bytesInOutput: 1 }; + }), + 'UNSUPPORTED_INPUT', + 'apps/cli/src/ghost.ts', + ], + // Metafile strings that escape the repository, are absolute or are not plain paths: no subject. + [input(`${'../'.repeat(12)}private-notes.js`), 'UNSUPPORTED_INPUT', undefined], + [input((c) => `../../..${c.root}/private-notes.js`), 'UNSUPPORTED_INPUT', undefined], + [input((c) => `${c.root}/private-notes.js`), 'UNSUPPORTED_INPUT', undefined], + [input('../../node_modules/x y/index.js'), 'UNSUPPORTED_INPUT', undefined], + [input(`../../${'x'.repeat(200)}.js`), 'UNSUPPORTED_INPUT', undefined], + [ + (c) => + editJson(join(c.dist, 'metafile-esm.json'), (metafile) => { + const extra = { imports: [], exports: [], inputs: {}, bytes: 1 }; + metafile.outputs[`${c.root}/x.js`] = extra; + metafile.outputs[`${c.root}/x.js.map`] = extra; + }), + 'OUTPUT_INVENTORY', + undefined, + ], + ]) { + const candidate = await fixture(); + change(candidate); + const error = refusal(candidate); + assert.ok(error instanceof CandidateFinalizationError, `${code}: ${error}`); + assert.equal( + error.message, + `CLI candidate finalization refused: ${code}` + + (subject === undefined ? '' : ` (${subject})`), + ); + assert.equal(error.message.includes(scratch), false); + } +}); + test('requires an empty private staging directory outside the candidate', async (t) => { const candidate = await fixture(); for (const [staging, code] of [ @@ -527,6 +844,8 @@ test('requires an empty private staging directory outside the candidate', async [join(scratch, 'absent-staging'), 'STAGING_PATH'], [join(candidate.root, 'apps'), 'STAGING_PATH'], [scratch, 'STAGING_PATH'], + [undefined, 'STAGING_PATH'], + [join(candidate.root, 'apps', 'cli', 'package.json'), 'STAGING_PATH'], ]) { assert.throws( () => finalizeCandidate({ repository: candidate.root, staging }), @@ -537,9 +856,22 @@ test('requires an empty private staging directory outside the candidate', async refusesBeforeWritingIgnoringStaging(candidate, 'STAGING_NOT_EMPTY'); rmSync(join(candidate.staging, 'left-over')); if (posixModes) { - chmodSync(candidate.staging, 0o755); - refusesBeforeWriting(candidate, 'STAGING_NOT_PRIVATE'); - chmodSync(candidate.staging, 0o700); + // A file outside the candidate is not a directory, however private its mode. + const file = join(scratch, `staging-file-${count}`); + writeFileSync(file, ''); + chmodSync(file, 0o700); + assert.throws( + () => finalizeCandidate({ repository: candidate.root, staging: file }), + (error) => error instanceof CandidateFinalizationError && error.code === 'STAGING_PATH', + ); + for (const mode of [0o755, 0o500]) { + chmodSync(candidate.staging, mode); + try { + refusesBeforeWriting(candidate, 'STAGING_NOT_PRIVATE'); + } finally { + chmodSync(candidate.staging, 0o700); + } + } // An owner-only directory another account owns refuses too, before anything is written. const realUid = process.getuid(); const uid = t.mock.method(process, 'getuid', () => realUid + 1); @@ -624,7 +956,7 @@ test('re-checks the finite closure and the migration copy on the finalized files ); }); -test('ordinary package and CLI builds stay map-free outside a named candidate', () => { +test('ordinary engine package builds stay map-free', () => { const root = new URL('../../', import.meta.url); for (const name of ['shared', 'llm', 'core', 'db', 'mcp']) { const path = fileURLToPath(new URL(`packages/${name}/tsconfig.build.json`, root)); @@ -636,13 +968,44 @@ test('ordinary package and CLI builds stay map-free outside a named candidate', assert.notEqual(options.inlineSourceMap, true, name); assert.notEqual(options.inlineSources, true, name); } - const config = readFileSync(new URL('apps/cli/tsup.config.ts', root), 'utf8'); - assert.match( - config, - /^const CANDIDATE = process\.env\.RELAVIUM_CLI_CANDIDATE_STAGING !== undefined;$/mu, +}); + +test('only a named candidate CLI build emits maps, and those maps embed no sources', () => { + // The real tsup config, compiled by tsup's own esbuild and evaluated beside a copy of the CLI + // manifest and installation, with and without the candidate variable. + const cli = fileURLToPath(new URL('../../apps/cli/', import.meta.url)); + const directory = join(scratch, 'tsup-config'); + mkdirSync(directory); + const source = readFileSync(join(cli, 'tsup.config.ts'), 'utf8'); + writeFileSync( + join(directory, 'config.mjs'), + transformSync(source, { loader: 'ts', format: 'esm' }).code, ); - assert.deepEqual(config.match(/^\s*sourcemap:.*$/gmu), [' sourcemap: CANDIDATE,']); - assert.match(config, /if \(CANDIDATE\) options\.sourcesContent = false;/u); + copyFileSync(join(cli, 'package.json'), join(directory, 'package.json')); + symlinkSync(join(cli, 'node_modules'), join(directory, 'node_modules'), 'junction'); + const evaluate = (staging) => { + const env = { ...process.env }; + delete env.RELAVIUM_CLI_CANDIDATE_STAGING; + if (staging !== undefined) env.RELAVIUM_CLI_CANDIDATE_STAGING = staging; + const script = `const { default: config } = await import(${JSON.stringify( + pathToFileURL(join(directory, 'config.mjs')).href, + )}); +const options = {}; +config.esbuildOptions?.(options, { format: 'esm' }); +process.stdout.write(JSON.stringify({ sourcemap: config.sourcemap, options }));`; + const result = spawnSync(process.execPath, ['--input-type=module', '--eval', script], { + env, + encoding: 'utf8', + timeout: 30_000, + }); + assert.equal(result.status, 0, result.stderr); + return JSON.parse(result.stdout); + }; + assert.deepEqual(evaluate(join(scratch, 'private-staging')), { + sourcemap: true, + options: { sourcesContent: false }, + }); + assert.deepEqual(evaluate(undefined), { sourcemap: false, options: {} }); }); test('keeps exact staged and public modes whatever the umask', { skip: !posixModes }, async () => { diff --git a/tools/cli-package/paths.mjs b/tools/cli-package/paths.mjs index 4c5d45f0..2519745e 100644 --- a/tools/cli-package/paths.mjs +++ b/tools/cli-package/paths.mjs @@ -1,4 +1,5 @@ -/** Path containment shared by the producer, the candidate tree and the finalizer. */ +/** Path containment shared by the producer, the candidate tree, the finalizer and the consumer. */ +import { realpathSync } from 'node:fs'; import { isAbsolute, relative, sep } from 'node:path'; /** @@ -10,3 +11,13 @@ export function within(parent, child) { const path = relative(parent, child); return path === '' || (path !== '..' && !path.startsWith(`..${sep}`) && !isAbsolute(path)); } + +/** + * Whether the existing `child` lies inside the existing `parent`, decided on the real paths and the + * letter case the filesystem stores (`realpathSync.native`), so neither a link nor another letter case + * of the same directory passes for elsewhere. + * @param {string} parent @param {string} child + */ +export function placedWithin(parent, child) { + return within(realpathSync.native(parent), realpathSync.native(child)); +} diff --git a/tools/cli-package/paths.test.mjs b/tools/cli-package/paths.test.mjs new file mode 100644 index 00000000..ce551cea --- /dev/null +++ b/tools/cli-package/paths.test.mjs @@ -0,0 +1,39 @@ +import assert from 'node:assert/strict'; +import { existsSync, mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, dirname, join } from 'node:path'; +import { after, test } from 'node:test'; +import { placedWithin, within } from './paths.mjs'; + +const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-paths-'))); +after(() => rmSync(scratch, { recursive: true, force: true })); + +test('containment is decided by whole path segments', () => { + assert.equal(within('/a/b', '/a/b'), true); + assert.equal(within('/a/b', '/a/b/c'), true); + // A segment that merely starts with two dots is inside. + assert.equal(within('/a/b', '/a/b/..c'), true); + assert.equal(within('/a/b', '/a'), false); + assert.equal(within('/a/b', '/a/c'), false); + assert.equal(within('/a/b', '/a/bc'), false); +}); + +test('placement compares the real paths and stored letter case of existing directories', () => { + const checkout = join(scratch, 'checkout'); + const inside = join(checkout, 'tmp'); + const outside = join(scratch, 'elsewhere'); + for (const directory of [inside, outside]) mkdirSync(directory, { recursive: true }); + assert.equal(placedWithin(checkout, inside), true); + assert.equal(placedWithin(checkout, outside), false); + // A link to the checkout is the checkout. + const alias = join(scratch, 'alias'); + symlinkSync(checkout, alias, process.platform === 'win32' ? 'junction' : 'dir'); + assert.equal(placedWithin(checkout, join(alias, 'tmp')), true); + assert.equal(placedWithin(alias, inside), true); + // On a case-insensitive filesystem, another letter case of the checkout is the checkout. + const variant = join(dirname(checkout), basename(checkout).toUpperCase()); + if (existsSync(variant)) { + assert.equal(placedWithin(checkout, join(variant, 'tmp')), true); + assert.equal(placedWithin(variant, inside), true); + } +}); diff --git a/tools/cli-package/pipeline.test.mjs b/tools/cli-package/pipeline.test.mjs index d54a0d5e..f4472d22 100644 --- a/tools/cli-package/pipeline.test.mjs +++ b/tools/cli-package/pipeline.test.mjs @@ -66,6 +66,13 @@ test('fresh pipeline orders five mapped dependency emits before the sole candida ]), ); assert.deepEqual(CANDIDATE_TSC_ARGS, commands[0].args); + // Each package's build state goes immediately before that package compiles, never after. + assert.deepEqual( + ports.calls + .slice(0, 10) + .map((call) => (call.remove ? `remove ${call.remove}` : `exec ${call.cwd}`)), + packages.flatMap((path) => [`remove ${path}`, `exec ${path}`]), + ); // Only the sole prepack learns the private staging directory. assert.ok(commands.slice(0, 5).every((call) => call.env === ports.env)); assert.deepEqual(commands[5].args, ['pack', '--pack-destination', ports.destination]); diff --git a/tools/cli-package/prepack.test.mjs b/tools/cli-package/prepack.test.mjs index ddc6e84e..56b5cabd 100644 --- a/tools/cli-package/prepack.test.mjs +++ b/tools/cli-package/prepack.test.mjs @@ -24,7 +24,7 @@ const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-prepack-orches after(() => rmSync(scratch, { recursive: true, force: true })); let count = 0; -function fixture({ buildExit = 0, tsup: installed = true } = {}) { +function fixture({ buildExit = 0, tsup: installed = true, finalizer } = {}) { count += 1; const root = join(scratch, `repository-${count}`); for (const path of [ @@ -44,6 +44,20 @@ function fixture({ buildExit = 0, tsup: installed = true } = {}) { join(root, 'node_modules'), process.platform === 'win32' ? 'junction' : 'dir', ); + if (finalizer !== undefined) { + // The real staging check and error class, with a stand-in for the finalization itself. + const tools = join(root, 'tools', 'cli-package'); + copyFileSync(join(tools, 'finalize.mjs'), join(tools, 'finalize-real.mjs')); + writeFileSync( + join(tools, 'finalize.mjs'), + `export { CANDIDATE_STAGING_ENV, CandidateFinalizationError, assertPrivateStaging } from './finalize-real.mjs';\n` + + `import { BundleClosureError } from '../bundle-closure/closure.mjs';\n` + + `import { AssertionError } from 'node:assert/strict';\n` + + `import { writeFileSync } from 'node:fs';\n` + + `export function finalizeCandidate(options) {\n${finalizer}\n}\n` + + `void BundleClosureError, AssertionError, writeFileSync;\n`, + ); + } const cli = join(root, 'apps', 'cli'); const tsup = join(cli, 'node_modules', 'tsup'); const marker = join(root, 'build-ran'); @@ -146,3 +160,59 @@ test('an unexpected error prints only its code, never its message or a host path assert.equal(result.stderr, '✗ CLI candidate finalization failed: MODULE_NOT_FOUND\n'); assert.equal(existsSync(candidate.marker), false); }); + +test('a staging directory inside the repository but outside the CLI refuses before the build', () => { + const candidate = fixture(); + const staging = join(candidate.root, `staging-${count}`); + mkdirSync(staging); + chmodSync(staging, 0o700); + const result = prepack(candidate, { staging }); + assert.equal(result.status, 1); + assert.match(result.stderr, /CLI candidate finalization refused: STAGING_PATH/u); + assert.equal(existsSync(candidate.marker), false); +}); + +test('a successful finalization gets the repository and staging and reports on stdout', () => { + const record = join(scratch, `finalizer-options-${count + 1}.json`); + const candidate = fixture({ + finalizer: + `writeFileSync(${JSON.stringify(record)}, JSON.stringify(options));\n` + + `return { outputs: [1, 2], maps: [1, 2], upstreamMaps: 5 };`, + }); + const staging = privateStaging(); + const result = prepack(candidate, { staging }); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(JSON.parse(readFileSync(record, 'utf8')), { + repository: candidate.root, + staging, + }); + // The pipeline counts exactly this line on stdout. + assert.equal( + (result.stdout.match(/^✓ CLI candidate finalized: /gmu) ?? []).length, + 1, + result.stdout, + ); + assert.match( + result.stdout, + /^✓ CLI candidate finalized: 2 output\(s\), 2 private map\(s\), 5 upstream map\(s\)$/mu, + ); +}); + +for (const [label, thrown, printed] of [ + [ + 'a closure refusal', + `throw new BundleClosureError('MAP_LINK', 'runtime output links a source map: dist/index.js');`, + '✗ CLI candidate closure: MAP_LINK: runtime output links a source map: dist/index.js\n', + ], + [ + 'an assertion', + `throw new AssertionError({ message: 'PACKED_CLI_SYNTHETIC_ASSERTION' });`, + '✗ CLI candidate finalization: PACKED_CLI_SYNTHETIC_ASSERTION\n', + ], +]) + test(`${label} prints its fixed line and nothing else`, () => { + const candidate = fixture({ finalizer: thrown }); + const result = prepack(candidate, { staging: privateStaging() }); + assert.equal(result.status, 1); + assert.equal(result.stderr, printed); + }); diff --git a/tools/cli-package/process.mjs b/tools/cli-package/process.mjs index 886a21aa..67f9488a 100644 --- a/tools/cli-package/process.mjs +++ b/tools/cli-package/process.mjs @@ -55,19 +55,21 @@ export function isolatedEnvironment(home) { * to the device; Node's `os.devNull` there (`\\.\nul`) reaches the C runtime, which refuses it, and * Git then exits rather than read no configuration. */ -const GIT_NULL_DEVICE = '/dev/null'; +export const GIT_NULL_DEVICE = '/dev/null'; /** * Git with no global or system configuration, for reading the original checkout: its own `.git/config` * and ignore rules still apply and a caller passes the developer's global excludes explicitly, but no * `.gitconfig`, `git/config`, `git/ignore` or `git/attributes` in a home, configuration or system - * location is read — not even one inside the checkout itself. Replace refs are ignored, so a status + * location is read — not even one inside the checkout itself (the callers also name the attributes and + * excludes files explicitly, so no default location applies either). Replace refs are ignored, so a status * compares the working tree with the commit HEAD names, never a substitute for it. */ export function isolatedGitEnvironment() { return { ...isolatedEnvironment(GIT_NULL_DEVICE), GIT_CONFIG_NOSYSTEM: '1', + GIT_ATTR_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: GIT_NULL_DEVICE, GIT_NO_REPLACE_OBJECTS: '1', }; @@ -218,7 +220,8 @@ export function forwardInterrupts() { * Run one tool child in its own process group (POSIX) and settle when it closed. Past `timeoutMs` its * group (on Windows its whole tree) is killed. On POSIX with a positive `graceMs` the group first gets * SIGTERM and `graceMs` more, so a child that stops its own detached children and removes its private - * directories on an interrupt (the producer, the consumer) does so before anything is killed. + * directories on an interrupt (the producer, the consumer) does so before anything is killed; the group + * is killed when the grace ends or as soon as the child closes, whichever comes first. */ export function runCommand(command, args, options = {}) { const { timeoutMs = 300_000, graceMs = 0, maxBytes = 8 * 1024 * 1024, ...spawnOptions } = options; @@ -275,10 +278,13 @@ export function runCommand(command, args, options = {}) { } try { process.kill(-child.pid, 'SIGTERM'); - } catch (error) { - if (error.code !== 'ESRCH') fault ??= error; + } catch { + // The group already exited; the timeout fault already stands. } - graceTimer = setTimeout(stop, graceMs); + graceTimer = setTimeout(() => { + graceTimer = undefined; + stop(); + }, graceMs); }, timeoutMs); for (const sink of ['stdout', 'stderr']) child[sink].on('data', (chunk) => { @@ -295,7 +301,16 @@ export function runCommand(command, args, options = {}) { closed = true; liveChildren.delete(live); clearTimeout(timer); - clearTimeout(graceTimer); + if (graceTimer !== undefined) { + // The leader exited on the SIGTERM before the grace ran out: a group member that ignored it + // is killed now, as it would have been when the grace ended. + clearTimeout(graceTimer); + try { + process.kill(-child.pid, 'SIGKILL'); + } catch (error) { + if (error.code !== 'ESRCH') fault ??= error; + } + } if (killer && killer.exitCode === null && killer.signalCode === null) await new Promise((done) => killer.once('close', done)); clearTimeout(killerTimer); diff --git a/tools/cli-package/process.test.mjs b/tools/cli-package/process.test.mjs index 505f889a..79be020f 100644 --- a/tools/cli-package/process.test.mjs +++ b/tools/cli-package/process.test.mjs @@ -14,9 +14,11 @@ import { import { homedir, tmpdir } from 'node:os'; import { join, resolve, sep } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { exitWithin } from './child-exit.mjs'; import { corepackHome, exerciseInstalledCommand, + forwardInterrupts, isolatedEnvironment, isolatedGitEnvironment, resolvePnpm, @@ -30,6 +32,7 @@ test('Git reading the original checkout gets no home, global or system configura assert.deepEqual(env, { ...isolatedEnvironment('/dev/null'), GIT_CONFIG_NOSYSTEM: '1', + GIT_ATTR_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: '/dev/null', GIT_NO_REPLACE_OBJECTS: '1', }); @@ -236,10 +239,9 @@ if (signal !== undefined) process.exit(128 + constants.signals[signal]); try { child = Number(await waitForFile(pidFile)); assert.ok(alive(child)); - const exited = new Promise((resolve) => tool.once('exit', resolve)); + const exited = exitWithin(tool); tool.kill('SIGINT'); - const bound = delay(20_000).then(() => 'still running'); - assert.equal(await Promise.race([exited, bound]), 130); + assert.deepEqual(await exited, [130, null]); assert.equal(readFileSync(marker, 'utf8'), 'SIGINT'); for (let waited = 0; alive(child); waited += 25) { assert.ok(waited < 5_000, 'the child outlived the forwarded interrupt'); @@ -280,9 +282,7 @@ console.log('not interrupted'); }); let stdout = ''; tool.stdout.on('data', (chunk) => (stdout += chunk)); - const [code, signal] = await new Promise((resolve) => - tool.once('exit', (exitCode, signalCode) => resolve([exitCode, signalCode])), - ); + const [code, signal] = await exitWithin(tool); assert.equal(signal, null); assert.equal(code, 130); assert.equal(stdout, ''); @@ -319,6 +319,8 @@ test('the corepack cache is computed as corepack computes it, and isolated child resolve('/xdg', 'node', 'corepack'), ); assert.equal(corepackHome({ LOCALAPPDATA: '/local' }), resolve('/local', 'node', 'corepack')); + // A relative location is resolved here, never by a child against its own working directory. + assert.equal(corepackHome({ COREPACK_HOME: 'relative/cache' }), resolve('relative/cache')); assert.equal( corepackHome({}), resolve( @@ -445,3 +447,172 @@ test( assert.ok(Date.now() - started >= 1500); }, ); + +test('an isolated child of an isolated tool keeps the corepack cache its caller named', async () => { + // check.mjs isolates the producer's HOME and caches in the temporary check root and names the + // developer's cache explicitly; the producer's own children must keep that cache. + const home = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-corepack-inherited-'))); + try { + const script = `import { isolatedEnvironment } from ${JSON.stringify(new URL('./process.mjs', import.meta.url).href)}; +console.log(isolatedEnvironment('/isolated-home').COREPACK_HOME);`; + const result = await runCommand(process.execPath, ['--input-type=module', '-e', script], { + env: { + PATH: process.env.PATH, + HOME: home, + XDG_CACHE_HOME: home, + COREPACK_HOME: '/developer/corepack', + }, + timeoutMs: 20_000, + }); + assert.equal(result.stdout.trim(), resolve('/developer/corepack')); + } finally { + rmSync(home, { recursive: true, force: true }); + } +}); + +/** + * Code for a group leader that starts a member in its own process group (not detached), records the + * member's pid and stays alive until a SIGTERM ends it; the member ignores SIGTERM. + */ +function groupWithStubbornMember(pidFile) { + const member = "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);"; + return ( + `const { spawn } = require('node:child_process');` + + `const member = spawn(process.execPath, ['-e', ${JSON.stringify(member)}], { stdio: 'ignore' });` + + `require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, String(member.pid));` + + 'setInterval(() => {}, 1000);' + ); +} + +test( + 'past its deadline the whole group is killed when the leader exits on the interrupt first', + { skip: process.platform === 'win32' }, + async (t) => { + const root = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-group-deadline-'))); + const pidFile = join(root, 'member.pid'); + t.after(() => { + const pid = existsSync(pidFile) ? Number(readFileSync(pidFile, 'utf8')) : NaN; + if (Number.isSafeInteger(pid) && pid > 0 && alive(pid)) process.kill(pid, 'SIGKILL'); + rmSync(root, { recursive: true, force: true }); + }); + await assert.rejects( + runCommand(process.execPath, ['-e', groupWithStubbornMember(pidFile)], { + timeoutMs: 1000, + graceMs: 4000, + }), + /PACKED_CLI_CHILD_TIMEOUT/u, + ); + // The leader exited on the SIGTERM; its member, which ignored it, goes with the settlement. + const member = Number(readFileSync(pidFile, 'utf8')); + for (let waited = 0; alive(member); waited += 25) { + assert.ok(waited < 2_000, 'a member of the group outlived its deadline'); + await delay(25); + } + }, +); + +test( + 'past its deadline every member of the group is interrupted, not only the leader', + { skip: process.platform === 'win32' }, + async (t) => { + const root = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-group-interrupt-'))); + const pidFile = join(root, 'member.pid'); + const marker = join(root, 'member-interrupted'); + t.after(() => { + const pid = existsSync(pidFile) ? Number(readFileSync(pidFile, 'utf8')) : NaN; + if (Number.isSafeInteger(pid) && pid > 0 && alive(pid)) process.kill(pid, 'SIGKILL'); + rmSync(root, { recursive: true, force: true }); + }); + const member = + `const fs = require('node:fs'); fs.writeFileSync(${JSON.stringify(pidFile)}, String(process.pid));` + + `process.on('SIGTERM', () => { fs.writeFileSync(${JSON.stringify(marker)}, 'SIGTERM'); process.exit(0); });` + + 'setInterval(() => {}, 1000);'; + // The leader ignores the interrupt, so nothing but the grace's end kills the group. + const leader = + `require('node:child_process').spawn(process.execPath, ['-e', ${JSON.stringify(member)}], { stdio: 'ignore' });` + + "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);"; + await assert.rejects( + runCommand(process.execPath, ['-e', leader], { timeoutMs: 2000, graceMs: 1500 }), + (error) => + /PACKED_CLI_CHILD_TIMEOUT/u.test(error.message) && error.result.signal === 'SIGKILL', + ); + assert.equal(readFileSync(marker, 'utf8'), 'SIGTERM'); + }, +); + +test( + 'without a grace a child past its deadline is killed outright, never interrupted first', + { skip: process.platform === 'win32' }, + async (t) => { + const root = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-no-grace-'))); + const marker = join(root, 'interrupted'); + t.after(() => rmSync(root, { recursive: true, force: true })); + await assert.rejects( + runCommand( + process.execPath, + [ + '-e', + `process.on('SIGTERM', () => require('node:fs').writeFileSync(${JSON.stringify(marker)}, 'SIGTERM'));` + + 'setInterval(() => {}, 1000);', + ], + { timeoutMs: 500 }, + ), + (error) => + /PACKED_CLI_CHILD_TIMEOUT/u.test(error.message) && error.result.signal === 'SIGKILL', + ); + assert.equal(existsSync(marker), false); + }, +); + +test('the grace is a non-negative whole number of milliseconds', () => { + for (const graceMs of [-1, 1.5, Number.NaN]) + assert.throws(() => runCommand(process.execPath, ['-e', ''], { graceMs }), String(graceMs)); +}); + +test( + 'an interrupt stops a child tree that ignores SIGTERM', + { skip: process.platform === 'win32' }, + async (t) => { + // The tool's children are killed outright: a build that traps SIGTERM cannot outlive it. + const root = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-stubborn-child-'))); + const pidFile = join(root, 'child.pid'); + t.after(() => { + const pid = existsSync(pidFile) ? Number(readFileSync(pidFile, 'utf8')) : NaN; + if (Number.isSafeInteger(pid) && pid > 0 && alive(pid)) process.kill(pid, 'SIGKILL'); + rmSync(root, { recursive: true, force: true }); + }); + const harness = ` +import { exitOnInterrupt, runCommand } from ${JSON.stringify(new URL('./process.mjs', import.meta.url).href)}; +await exitOnInterrupt(() => {}, () => + runCommand(process.execPath, ['-e', ${JSON.stringify( + `require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, String(process.pid));` + + "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);", + )}], { timeoutMs: 60_000 }), +); +`; + const tool = spawn(process.execPath, ['--input-type=module', '-e', harness], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + const child = Number(await waitForFile(pidFile)); + const exited = exitWithin(tool); + tool.kill('SIGINT'); + assert.deepEqual(await exited, [130, null]); + for (let waited = 0; alive(child); waited += 25) { + assert.ok(waited < 3_000, 'the child outlived the interrupt'); + await delay(25); + } + }, +); + +test('forwarding interrupts adds one listener per signal and disposing removes them', async () => { + const signals = ['SIGINT', 'SIGTERM', 'SIGHUP']; + const counts = () => signals.map((signal) => process.listenerCount(signal)); + const before = counts(); + const interrupts = forwardInterrupts(); + assert.deepEqual( + counts(), + before.map((count) => count + 1), + ); + await interrupts.dispose(); + assert.deepEqual(counts(), before); +}); diff --git a/tools/cli-package/produce.mjs b/tools/cli-package/produce.mjs index f47ec90a..ece697e4 100644 --- a/tools/cli-package/produce.mjs +++ b/tools/cli-package/produce.mjs @@ -30,12 +30,15 @@ import { emitFreshPackage } from './pipeline.mjs'; * Produce one package-test artifact from the original checkout `repository` into the empty * `destination`. The candidate tree and its private staging are siblings in one temporary root, * owner-only where POSIX modes are observable, that shares no path with the artifact directory or the - * checkout. That root and the build home are removed whether production succeeds or fails, each - * attempted even when the other's removal fails. An interrupt (SIGINT, SIGTERM, SIGHUP) stops the - * build's child trees, removes both directories and empties the artifact directory, which was empty - * at the start, before the process exits with the signal's code — also when it arrives after the last - * child exited, during the seal or during the final removal (`exitOnInterrupt`). A step of that - * cleanup that fails leaves one fixed line, never a path, and the exit code stays the signal's. + * checkout. That root and the build home are removed whether production succeeds or fails — the + * staging directory, which holds the private maps, first — each removal attempted even when another + * fails. A failed production keeps its own error, also empties the artifact directory, which was empty + * at the start, and reports a failed removal only as one fixed line. An interrupt (SIGINT, SIGTERM, + * SIGHUP) stops the build's child trees, removes the staging directory, empties the artifact directory + * and only then removes the candidate tree and the build home, before the process exits with the + * signal's code — also when it arrives after the last child exited, during the seal or during the + * final removal (`exitOnInterrupt`). A step of that cleanup that fails leaves one fixed line, never a + * path, and the exit code stays the signal's. * @param {{repository: string, destination: string, environment: Record, * temporary?: string, ports?: {prepare?: typeof prepareCandidate, emit?: typeof emitFreshPackage, * seal?: typeof sealPackage, noBuildCache?: typeof assertNoBuildCache, @@ -63,52 +66,58 @@ export async function producePackage({ assert.deepEqual(readdirSync(destination), [], 'PACKED_CLI_FRESH_DESTINATION_REQUIRED'); const home = realpathSync.native(mkdtempSync(join(temporary, 'relavium-package-build-'))); let root; - // Every removal is attempted even when an earlier one fails; the first failure is rethrown. - const discard = () => { + /** Remove each of `paths`, attempting every one; the first failure, if any. */ + const removeAll = (paths) => { let failure; - for (const path of [root, home]) { - if (path === undefined) continue; + for (const path of paths) { try { remove(path); } catch (error) { failure ??= error; } } - if (failure !== undefined) throw failure; + return failure; }; - // An interrupt's cleanup never throws: a failed step leaves one fixed line, without a path, and - // the producer still exits with the signal's code. - const interrupted = () => { - let failed = false; - const attempt = (step) => { - try { - step(); - } catch { - failed = true; - } - }; - attempt(discard); - let names = []; - attempt(() => { + const staging = () => (root === undefined ? [] : [join(root, 'staging')]); + const rest = () => (root === undefined ? [home] : [root, home]); + /** Empty the artifact directory, which was empty at the start; the first failure, if any. */ + const emptyArtifact = () => { + let names; + try { names = readdirSync(destination); - }); - for (const name of names) { - attempt(() => - rmSync(join(destination, name), { recursive: true, force: true, maxRetries: 3 }), - ); + } catch (error) { + return error; } - if (failed) - attempt(() => writeSync(2, '✗ CLI package production interrupted: CLEANUP_FAILED\n')); + return removeAll(names.map((name) => join(destination, name))); + }; + /** One fixed diagnostic line; a failed write cannot replace the outcome it reports. */ + const report = (line) => { + try { + writeSync(2, line); + } catch { + // Nothing else can carry it. + } + }; + // An interrupt's cleanup never throws: a failed step leaves one fixed line, without a path, and + // the producer still exits with the signal's code. The maps go first, then the artifact directory, + // then the large candidate tree and the build home. + const interrupted = () => { + const failures = [removeAll(staging()), emptyArtifact(), removeAll(rest())]; + if (failures.some((failure) => failure !== undefined)) + report('✗ CLI package production interrupted: CLEANUP_FAILED\n'); }; return exitOnInterrupt(interrupted, async () => { + let failed = false; + let failure; + let produced; try { root = realpathSync.native(mkdtempSync(join(temporary, 'relavium-package-candidate-'))); chmodSync(root, 0o700); const candidate = join(root, 'source'); - const staging = join(root, 'staging'); + const [privateStaging] = staging(); mkdirSync(candidate, { mode: 0o700 }); - mkdirSync(staging, { mode: 0o700 }); - chmodSync(staging, 0o700); + mkdirSync(privateStaging, { mode: 0o700 }); + chmodSync(privateStaging, 0o700); // Native real paths carry the letter case the filesystem stores, so no alias passes for elsewhere. const published = realpathSync.native(destination); const checkout = realpathSync.native(repository); @@ -125,7 +134,7 @@ export async function producePackage({ const receipt = await emit({ repository: candidate, destination, - staging, + staging: privateStaging, env, pnpm, removeBuildState: (cwd) => { @@ -136,17 +145,32 @@ export async function producePackage({ noBuildCache(candidate); return seal(artifactDirectory, { repository: candidate, - staging, + staging: privateStaging, source: identity.source, dirty: identity.dirty, }); }, }); - return { receipt, identity }; - } finally { - // The interrupt handler is still installed: an interrupt during this removal is honoured once - // it is done, never in the middle of it. - discard(); + produced = { receipt, identity }; + } catch (error) { + failed = true; + failure = error; + } + // The interrupt handler is still installed: an interrupt during this removal is honoured once it + // is done, never in the middle of it. + const removal = removeAll([...staging(), ...rest()]); + if (failed) { + // A failed production leaves no artifact and keeps its own error. + const emptied = emptyArtifact(); + if (removal !== undefined || emptied !== undefined) + report('✗ CLI package production cleanup failed: CLEANUP_FAILED\n'); + throw failure; + } + if (removal !== undefined) { + // Without its cleanup the production failed: no artifact remains, and the removal's error is it. + emptyArtifact(); + throw removal; } + return produced; }); } diff --git a/tools/cli-package/produce.test.mjs b/tools/cli-package/produce.test.mjs index e24459ef..d70c1045 100644 --- a/tools/cli-package/produce.test.mjs +++ b/tools/cli-package/produce.test.mjs @@ -9,13 +9,15 @@ import { realpathSync, rmSync, statSync, + symlinkSync, writeFileSync, } from 'node:fs'; import { tmpdir } from 'node:os'; -import { basename, dirname, join } from 'node:path'; +import { basename, dirname, join, sep } from 'node:path'; import { after, test } from 'node:test'; import { setTimeout as delay } from 'node:timers/promises'; import { assertNoBuildCache } from './candidate.mjs'; +import { exitWithin } from './child-exit.mjs'; import { isolatedEnvironment } from './process.mjs'; import { producePackage } from './produce.mjs'; import { productionFailure } from './production-failure.mjs'; @@ -137,6 +139,9 @@ test('the producer builds and seals only the owned candidate, never the original assert.deepEqual(seen.prepare.env, isolatedEnvironment(home)); assert.deepEqual(seen.emit.env, seen.prepare.env); assert.deepEqual(seen.pnpm.env, seen.prepare.env); + // pnpm's version is checked in the original checkout, and the emit gets that same pnpm port. + assert.equal(seen.pnpm.cwd, repository); + assert.equal(seen.emit.pnpm, fakes.pnpm); for (const key of Object.keys(PLANTED)) assert.equal(key in seen.prepare.env, false); const candidate = seen.prepare.candidate; // The candidate and its staging are siblings in one private root outside the checkout and the artifact. @@ -168,6 +173,25 @@ test('the producer builds and seals only the owned candidate, never the original assert.deepEqual(readdirSync(destination), ['relavium-0.1.1.tgz']); }); +test('without a named temporary directory the producer works in the system one', async (t) => { + const repository = fresh('checkout-system-temporary'); + const destination = join(fresh('artifact-parent'), 'artifact'); + const temporary = fresh('system-tmp'); + const names = ['TMPDIR', 'TMP', 'TEMP']; + const saved = names.map((name) => [name, process.env[name]]); + t.after(() => { + for (const [name, value] of saved) + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + }); + for (const name of names) process.env[name] = temporary; + const { seen, ports: fakes } = ports(); + await producePackage({ repository, destination, environment: {}, ports: fakes }); + assert.equal(dirname(seen.prepare.home), temporary); + assert.equal(dirname(dirname(seen.prepare.candidate)), temporary); + assert.deepEqual(leftovers(temporary), []); +}); + test('another pnpm than the pinned one refuses before anything is prepared', async () => { const temporary = fresh('tmp'); const { seen, ports: fakes } = ports(); @@ -238,11 +262,12 @@ test('a dirty source identity reaches the seal and the returned identity', async test('a build cache left in the candidate refuses before the seal, and nothing private remains', async () => { const temporary = fresh('tmp'); + const destination = join(fresh('artifact-parent'), 'artifact'); const { seen, ports: fakes } = ports({ plantCache: true }); await assert.rejects( producePackage({ repository: fresh('original-checkout'), - destination: join(fresh('artifact-parent'), 'artifact'), + destination, environment: {}, temporary, ports: fakes, @@ -251,6 +276,7 @@ test('a build cache left in the candidate refuses before the seal, and nothing p ); assert.equal(seen.seal, undefined); assert.deepEqual(leftovers(temporary), []); + assert.deepEqual(readdirSync(destination), []); }); test('a failure inside the build still removes the private root', async () => { @@ -332,6 +358,50 @@ test('private staging never shares a path with the checkout or the artifact', as assert.equal(cased.prepare, undefined); assert.deepEqual(leftovers(inside), []); } + // A link to the checkout or to the artifact directory is that directory too. + const linkKind = process.platform === 'win32' ? 'junction' : 'dir'; + const checkoutAlias = join(fresh('alias-parent'), 'checkout'); + symlinkSync(repository, checkoutAlias, linkKind); + await assert.rejects( + producePackage({ + repository: checkoutAlias, + destination: join(fresh('artifact-parent'), 'artifact'), + environment: {}, + temporary: inside, + ports: ports().ports, + }), + /PRIVATE_STAGING_PLACEMENT/u, + ); + const realArtifact = fresh('real-artifact'); + const artifactAlias = join(fresh('alias-parent'), 'artifact'); + symlinkSync(realArtifact, artifactAlias, linkKind); + await assert.rejects( + producePackage({ + repository, + destination: artifactAlias, + environment: {}, + temporary: realArtifact, + ports: ports().ports, + }), + /PRIVATE_STAGING_PLACEMENT/u, + ); + assert.deepEqual(readdirSync(realArtifact), []); + // On a case-insensitive filesystem, another letter case of the artifact directory is that directory. + const casedArtifact = fresh('cased-artifact'); + const casedVariant = join(dirname(casedArtifact), basename(casedArtifact).toUpperCase()); + if (existsSync(casedVariant)) { + await assert.rejects( + producePackage({ + repository, + destination: casedVariant, + environment: {}, + temporary: casedArtifact, + ports: ports().ports, + }), + /PRIVATE_STAGING_PLACEMENT/u, + ); + assert.deepEqual(readdirSync(casedArtifact), []); + } // So would a temporary directory inside the artifact directory. const artifact = join(fresh('artifact-parent'), 'artifact'); mkdirSync(artifact); @@ -360,10 +430,10 @@ test('private staging never shares a path with the checkout or the artifact', as }); /** A producer run whose build is a long-lived child, as `pnpm pack` is, behind fake ports. */ -function interruptedProducer(temporary, destination, pidFile, grandchildFile) { +function interruptedProducer(temporary, destination, pidFile, grandchildFile, removalLog) { const url = (name) => JSON.stringify(new URL(name, import.meta.url).href); const script = ` -import { writeFileSync } from 'node:fs'; +import { appendFileSync, mkdirSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { producePackage } from ${url('./produce.mjs')}; import { isolatedEnvironment, runCommand } from ${url('./process.mjs')}; @@ -379,6 +449,8 @@ await producePackage({ emit: async ({ staging, destination }) => { writeFileSync(join(staging, 'index.js.map'), 'private map'); writeFileSync(join(destination, 'relavium-0.1.1.tgz'), 'unsealed'); + mkdirSync(join(destination, 'nested')); + writeFileSync(join(destination, 'nested', 'part'), 'part'); await runCommand(process.execPath, ['-e', ${JSON.stringify( `const { spawn } = require('node:child_process');` + `const { writeFileSync } = require('node:fs');` + @@ -390,6 +462,13 @@ await producePackage({ `setInterval(() => {}, 1000);`, )}], { env: isolatedEnvironment(${JSON.stringify(temporary)}), timeoutMs: 60_000 }); }, + // Records each removal and how many entries the artifact directory still had at that moment. + remove: (path) => { + const entry = path.startsWith(${JSON.stringify(destination)}); + const kind = entry ? 'artifact' : path.endsWith('staging') ? 'staging' : path.includes('relavium-package-candidate-') ? 'candidate' : 'build'; + appendFileSync(${JSON.stringify(removalLog)}, (entry ? kind : kind + ':' + readdirSync(${JSON.stringify(destination)}).length) + '\\n'); + rmSync(path, { recursive: true, force: true }); + }, }, }); `; @@ -422,7 +501,16 @@ for (const [signal, code] of [ const pids = fresh('pid'); const pidFile = join(pids, 'sleeper.pid'); const grandchildFile = join(pids, 'grandchild.pid'); - const child = interruptedProducer(temporary, destination, pidFile, grandchildFile); + const removalLog = join(pids, 'removals'); + const child = interruptedProducer( + temporary, + destination, + pidFile, + grandchildFile, + removalLog, + ); + let stderr = ''; + child.stderr.on('data', (chunk) => (stderr += chunk)); const tree = []; try { for (const file of [pidFile, grandchildFile]) { @@ -437,9 +525,9 @@ for (const [signal, code] of [ tree.push(Number(readFileSync(file, 'utf8'))); } assert.ok(tree.every(alive)); - const exited = new Promise((resolve) => child.once('exit', resolve)); + const exited = exitWithin(child); child.kill(signal); - assert.equal(await exited, code); + assert.deepEqual(await exited, [code, null]); // The whole build tree is stopped, not only the process the producer started. for (let waited = 0; tree.some(alive); waited += 25) { assert.ok(waited < 5_000, 'part of the build tree outlived the interrupt'); @@ -447,6 +535,13 @@ for (const [signal, code] of [ } assert.deepEqual(leftovers(temporary), []); assert.deepEqual(readdirSync(destination), []); + // The maps go first, then the artifact directory (its nested entry too), then the large + // candidate tree and the build home; a clean interrupt prints nothing. + assert.equal( + readFileSync(removalLog, 'utf8'), + 'staging:2\nartifact\nartifact\ncandidate:0\nbuild:0\n', + ); + assert.equal(stderr, ''); } finally { if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); for (const pid of tree) if (alive(pid)) process.kill(pid, 'SIGKILL'); @@ -457,31 +552,141 @@ for (const [signal, code] of [ test('a failed removal still removes the other private directory and rethrows', async () => { const temporary = fresh('tmp'); + const destination = join(fresh('artifact-parent'), 'artifact'); const { ports: fakes } = ports(); const refusal = Object.assign(new Error('EACCES: synthetic refusal'), { code: 'EACCES' }); fakes.remove = (path) => { - if (path.includes('relavium-package-candidate-')) throw refusal; + if (path.includes('relavium-package-candidate-') && !path.endsWith('staging')) throw refusal; rmSync(path, { recursive: true, force: true }); }; const before = listeners(); await assert.rejects( producePackage({ repository: fresh('original-checkout'), - destination: join(fresh('artifact-parent'), 'artifact'), + destination, environment: {}, temporary, ports: fakes, }), (error) => error === refusal, ); - // The build home is gone although the private root's removal failed first. + // The build home is gone although the private root's removal failed first, and without its + // cleanup the production leaves no artifact. assert.deepEqual( leftovers(temporary).map((name) => name.replace(/-[^-]+$/u, '')), ['relavium-package-candidate'], ); + assert.deepEqual(readdirSync(destination), []); assert.deepEqual(listeners(), before); }); +test('every removal is attempted and the first failure is the one rethrown', async () => { + const temporary = fresh('tmp'); + const destination = join(fresh('artifact-parent'), 'artifact'); + const { ports: fakes } = ports(); + const first = new Error('synthetic staging refusal'); + const second = new Error('synthetic candidate refusal'); + const removed = []; + const kind = (path) => { + if (path.startsWith(destination)) return 'artifact'; + if (path.endsWith('staging')) return 'staging'; + return path.includes('relavium-package-candidate-') ? 'candidate' : 'build'; + }; + fakes.remove = (path) => { + removed.push(kind(path)); + if (kind(path) === 'staging') throw first; + if (kind(path) === 'candidate') throw second; + rmSync(path, { recursive: true, force: true }); + }; + await assert.rejects( + producePackage({ + repository: fresh('original-checkout'), + destination, + environment: {}, + temporary, + ports: fakes, + }), + (error) => error === first, + ); + // The staging directory, which holds the private maps, goes first; without its cleanup the + // production then leaves no artifact. + assert.deepEqual(removed, ['staging', 'candidate', 'build', 'artifact']); +}); + +test('a failed production keeps its own error and leaves no artifact', async () => { + const temporary = fresh('tmp'); + const destination = join(fresh('artifact-parent'), 'artifact'); + const { ports: fakes } = ports(); + const refusal = new Error('CLI package archive refused: EXTRA_FILE'); + fakes.seal = async () => { + throw refusal; + }; + await assert.rejects( + producePackage({ + repository: fresh('original-checkout'), + destination, + environment: {}, + temporary, + ports: fakes, + }), + (error) => error === refusal, + ); + // The fake emit wrote the tarball before the seal refused it. + assert.deepEqual(readdirSync(destination), []); + assert.deepEqual(leftovers(temporary), []); +}); + +for (const [label, { cleanupFails, closeStderr }, stderr] of [ + [ + 'whose cleanup also fails keeps its own error and adds one fixed line', + { cleanupFails: true, closeStderr: false }, + '✗ CLI package production cleanup failed: CLEANUP_FAILED\n', + ], + ['whose cleanup succeeds keeps its own error and adds no line', { cleanupFails: false }, ''], + // A line that cannot be written never replaces the error it would have reported. + [ + 'keeps its own error even when the cleanup line cannot be written', + { cleanupFails: true, closeStderr: true }, + '', + ], +]) { + test(`a failed production ${label}`, { skip: !posixModes }, () => { + const temporary = fresh('tmp'); + mkdirSync(join(temporary, 'checkout')); + const url = (name) => JSON.stringify(new URL(name, import.meta.url).href); + const script = ` +import { closeSync, rmSync } from 'node:fs'; +import { producePackage } from ${url('./produce.mjs')}; +if (${closeStderr === true}) closeSync(2); +try { + await producePackage({ + repository: ${JSON.stringify(join(temporary, 'checkout'))}, + destination: ${JSON.stringify(join(fresh('artifact-parent'), 'artifact'))}, + environment: {}, + temporary: ${JSON.stringify(temporary)}, + ports: { + pnpm: async () => ({ stdout: '9.12.3\\n', stderr: '' }), + excludesFile: async () => undefined, + prepare: async () => { throw new Error('PACKED_CLI_SOURCE_CHANGED_DURING_COPY'); }, + remove: (path) => { + if (${cleanupFails} && path.includes('relavium-package-candidate-')) throw Object.assign(new Error('EBUSY: ' + path), { code: 'EBUSY' }); + rmSync(path, { recursive: true, force: true }); + }, + }, + }); +} catch (error) { + console.log('REJECTED:' + error.message); +} +`; + const result = spawnSync(process.execPath, ['--input-type=module', '-e', script], { + encoding: 'utf8', + timeout: 20_000, + }); + assert.equal(result.stdout, 'REJECTED:PACKED_CLI_SOURCE_CHANGED_DURING_COPY\n'); + assert.equal(result.stderr, stderr); + }); +} + test( 'a failed removal during an interrupt leaves one fixed line and still exits with the signal', { skip: !posixModes }, @@ -512,7 +717,7 @@ await producePackage({ )}], { env: isolatedEnvironment(${JSON.stringify(temporary)}), timeoutMs: 60_000 }); }, remove: (path) => { - if (path.includes('relavium-package-candidate-')) + if (path.includes('relavium-package-candidate-') && !path.endsWith('staging')) throw Object.assign(new Error('EACCES: permission denied, unlink ' + JSON.stringify(path)), { code: 'EACCES' }); rmSync(path, { recursive: true, force: true }); }, @@ -535,13 +740,15 @@ await producePackage({ await delay(25); } build = Number(readFileSync(pidFile, 'utf8')); - const exited = new Promise((resolve) => - child.once('exit', (code, signal) => resolve([code, signal])), - ); + const exited = exitWithin(child); child.kill('SIGINT'); assert.deepEqual(await exited, [130, null]); assert.equal(stderr, '✗ CLI package production interrupted: CLEANUP_FAILED\n'); - assert.equal(alive(build), false); + // The build was killed; its exit is reaped a moment later. + for (let waited = 0; alive(build); waited += 25) { + assert.ok(waited < 5_000, 'the build outlived the interrupt'); + await delay(25); + } // The other steps still ran: the build home is gone and the artifact directory is empty. assert.deepEqual( leftovers(temporary).map((name) => name.replace(/-[^-]+$/u, '')), @@ -617,16 +824,18 @@ for (const during of ['seal', 'removal']) const child = lateInterrupt(temporary, destination, during); let stdout = ''; child.stdout.on('data', (chunk) => (stdout += chunk)); - const [code, signal] = await new Promise((resolve) => - child.once('exit', (exitCode, signalCode) => resolve([exitCode, signalCode])), - ); - // Honoured once the synchronous work is done: the removal completes, the artifact is emptied - // and the producer exits with the signal's code instead of reporting success. - assert.equal(signal, null); - assert.equal(code, 130); - assert.equal(stdout, ''); - assert.deepEqual(leftovers(temporary), []); - assert.deepEqual(readdirSync(destination), []); + try { + const [code, signal] = await exitWithin(child); + // Honoured once the synchronous work is done: the removal completes, the artifact is + // emptied and the producer exits with the signal's code instead of reporting success. + assert.equal(signal, null); + assert.equal(code, 130); + assert.equal(stdout, ''); + assert.deepEqual(leftovers(temporary), []); + assert.deepEqual(readdirSync(destination), []); + } finally { + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); + } }, ); @@ -685,3 +894,80 @@ test('the producer entry point prints only the fixed failure lines and exits 1', ); assert.equal(result.stderr.includes(QUOTED_MAP), false); }); + +test('the failure report bounds what it forwards and forwards nothing but anchored diagnostics', () => { + const failed = (stdout, stderr) => + Object.assign(new Error('PACKED_CLI_CHILD_FAILED: 1'), { result: { stdout, stderr } }); + const code = '✗ CLI package production failed: PACKED_CLI_CHILD_FAILED'; + // At most 240 printable characters after the prefix. + const longest = `✗ CLI ${'a'.repeat(240)}`; + assert.deepEqual(productionFailure(failed('', `${longest}\n✗ CLI ${'a'.repeat(241)}\n`)), [ + longest, + code, + ]); + // A line that merely ends in the prefix's text, after quoted map content, is not a diagnostic. + assert.deepEqual( + productionFailure(failed('', `1 | {"names":["${QUOTED_MAP}"],"x":"✗ CLI tail"}\n`)), + [code], + ); + // At most eight diagnostics, from stdout as well as stderr, trimmed. + const twelve = Array.from({ length: 12 }, (_, index) => `✗ CLI line ${index}`); + assert.deepEqual(productionFailure(failed(` ${twelve.join('\n')}`, '')), [ + ...twelve.slice(0, 8), + code, + ]); + // A system code must look like one; anything else is unexpected. + for (const value of ['/private/host/path', 'ERR with space', 1, 'lower']) + assert.deepEqual(productionFailure(Object.assign(new Error('x'), { code: value })), [ + '✗ CLI package production failed: UNEXPECTED', + ]); +}); + +test('the producer entry point hands the checkout, the artifact directory and its own environment over', () => { + const root = fresh('entry-wiring'); + const tools = join(root, 'tools', 'cli-package'); + mkdirSync(tools, { recursive: true }); + for (const name of ['pack.mjs', 'production-failure.mjs']) + writeFileSync(join(tools, name), readFileSync(new URL(name, import.meta.url))); + writeFileSync( + join(tools, 'produce.mjs'), + `import { writeFileSync } from 'node:fs';\n` + + `export async function producePackage({ repository, destination, environment }) {\n` + + ` writeFileSync(process.env.RELAVIUM_ENTRY_MARKER, JSON.stringify({ repository, destination, marker: environment.RELAVIUM_ENTRY_MARKER, same: environment === process.env }));\n` + + ` return { receipt: { mode: 'compiled_package_test', source: 's', tarballSha256: 'h', files: [1, 2] }, identity: { files: 3 } };\n` + + `}\n`, + ); + const cwd = fresh('entry-cwd'); + const marker = join(fresh('entry-marker'), 'called'); + const env = { PATH: process.env.PATH, RELAVIUM_ENTRY_MARKER: marker }; + const run = (args) => + spawnSync(process.execPath, [join(tools, 'pack.mjs'), ...args], { + cwd, + env, + encoding: 'utf8', + timeout: 20_000, + }); + const result = run(['out']); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(JSON.parse(readFileSync(marker, 'utf8')), { + repository: `${root}${sep}`, + destination: join(cwd, 'out'), + marker, + same: true, + }); + assert.deepEqual(JSON.parse(result.stdout), { + mode: 'compiled_package_test', + source: 's', + sha256: 'h', + files: 2, + candidateFiles: 3, + privateMaps: 'staged privately, then discarded', + symbolication: 'not implemented until Wave 3 Step 9a.2', + }); + // Exactly one artifact directory: any other count refuses before the producer runs. + rmSync(marker); + for (const args of [[], ['one', 'two']]) { + assert.notEqual(run(args).status, 0, String(args.length)); + assert.equal(existsSync(marker), false); + } +}); diff --git a/tools/cli-package/release-guard.test.mjs b/tools/cli-package/release-guard.test.mjs index 7e8bec54..a8dc5989 100644 --- a/tools/cli-package/release-guard.test.mjs +++ b/tools/cli-package/release-guard.test.mjs @@ -109,3 +109,31 @@ test('the guard refuses when its rule cannot be loaded', () => { assert.equal(run.stdout, ''); assert.equal(run.stderr, '::error::CLI publication refused: GUARD_FAILURE\n'); }); + +for (const [label, rule, stderr] of [ + ['returns', '', '::error::CLI publication refused: GUARD_FAILURE\n'], + [ + 'throws an error that is not its own', + "const error = new Error('x'); error.code = 'ENOENT'; throw error;", + '::error::CLI publication refused: GUARD_FAILURE\n', + ], + // Whatever the rule does to the process, the exit code was already 1 before it ran. + ['exits the process itself', 'process.exit();', ''], +]) { + test(`the guard refuses when its rule ${label}`, () => { + const directory = join(scratch, `rule-${label.replaceAll(' ', '-')}`); + mkdirSync(directory); + copyFileSync(guard, join(directory, 'release-guard.mjs')); + writeFileSync( + join(directory, 'release-guard-core.mjs'), + 'export class ReleaseGuardError extends Error {\n' + + ' constructor(code) { super(code); this.code = code; }\n' + + '}\n' + + `export function requireReleaseCandidate() { ${rule} }\n`, + ); + const run = runGuard(join(directory, 'release-guard.mjs'), [artifact(PACKAGE_TEST)]); + assert.equal(run.status, 1); + assert.equal(run.stdout, ''); + assert.equal(run.stderr, stderr); + }); +} diff --git a/tools/cli-package/seal.test.mjs b/tools/cli-package/seal.test.mjs index 00c89704..6ebff51d 100644 --- a/tools/cli-package/seal.test.mjs +++ b/tools/cli-package/seal.test.mjs @@ -1,10 +1,23 @@ import assert from 'node:assert/strict'; +import { Buffer } from 'node:buffer'; import { createHash } from 'node:crypto'; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { test } from 'node:test'; -import { finalizedPublicFiles, producerNodeMajor } from './seal.mjs'; +import { fileURLToPath } from 'node:url'; +import { gzipSync } from 'node:zlib'; +import { finalizedPublicFiles, producerNodeMajor, sealPackage } from './seal.mjs'; +import { ustar } from './ustar-fixture.mjs'; /** A finalized candidate: public JS only in dist, its metafile and hash record in private staging. */ function withFinalizedFixture(body) { @@ -219,3 +232,233 @@ test( }); }, ); + +const repository = fileURLToPath(new URL('../..', import.meta.url)); +const SOURCE = 'a'.repeat(40); +const ORIGIN_VARIABLES = [ + 'GITHUB_ACTIONS', + 'GITHUB_REPOSITORY', + 'GITHUB_WORKFLOW', + 'GITHUB_RUN_ID', + 'GITHUB_RUN_ATTEMPT', + 'GITHUB_EVENT_NAME', + 'GITHUB_SHA', + 'RELAVIUM_PACKAGE_SOURCE_SHA', +]; +const GITHUB_ORIGIN = { + GITHUB_ACTIONS: 'true', + GITHUB_REPOSITORY: 'owner/name', + GITHUB_WORKFLOW: 'CI', + GITHUB_RUN_ID: '42', + GITHUB_RUN_ATTEMPT: '1', + GITHUB_EVENT_NAME: 'push', + GITHUB_SHA: 'b'.repeat(40), + RELAVIUM_PACKAGE_SOURCE_SHA: SOURCE, +}; + +/** Run `body` with exactly `variables` as the origin environment, restoring the job's own after. */ +async function withOrigin(variables, body) { + const saved = ORIGIN_VARIABLES.map((name) => [name, process.env[name]]); + for (const name of ORIGIN_VARIABLES) delete process.env[name]; + Object.assign(process.env, variables); + try { + return await body(); + } finally { + for (const [name, value] of saved) + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } +} + +/** + * A finalized candidate that is not the repository holding seal.mjs: its own toolchain pins, + * catalog, CLI manifest, migrations and documents, a private staging directory outside the + * candidate root, and pnpm's tarball of exactly the finalized public files. + */ +async function withCandidate(body, { packageManager = 'pnpm@9.12.3' } = {}) { + const temporary = mkdtempSync(join(tmpdir(), 'relavium-seal-candidate-')); + const root = join(temporary, 'candidate'); + const staging = join(temporary, 'private-staging'); + const artifact = join(temporary, 'artifact'); + const write = (path, bytes) => { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, bytes); + }; + const published = { + name: 'relavium', + version: '0.1.1', + type: 'module', + bin: { relavium: 'dist/index.js' }, + engines: { node: '>=22' }, + scripts: { start: 'node dist/index.js' }, + dependencies: { commander: '^14.0.0' }, + }; + const js = { + 'dist/index.js': '#!/usr/bin/env node\nimport "./chunk-ABC.js";\n', + 'dist/chunk-ABC.js': 'export {};\n', + }; + const migrations = ['0000_first.sql', 'meta/_journal.json', 'meta/0000_snapshot.json']; + try { + write(join(root, '.nvmrc'), `${process.versions.node.split('.')[0]}\n`); + write(join(root, 'package.json'), JSON.stringify({ packageManager })); + write(join(root, 'pnpm-workspace.yaml'), 'packages: []\ncatalog:\n commander: ^14.0.0\n'); + write(join(root, 'LICENSE'), 'candidate license\n'); + write( + join(root, 'apps/cli/package.json'), + JSON.stringify({ + ...published, + scripts: { ...published.scripts, prepack: 'node prepack.mjs' }, + dependencies: { commander: 'catalog:' }, + }), + ); + // The copied tree's own installation resolves `yaml` for the catalog read. + symlinkSync( + join(repository, 'apps/cli/node_modules'), + join(root, 'apps/cli/node_modules'), + 'junction', + ); + write(join(root, 'apps/cli/README.md'), 'candidate readme\n'); + write(join(root, 'apps/cli/CHANGELOG.md'), 'candidate changes\n'); + for (const [path, text] of Object.entries(js)) write(join(root, 'apps/cli', path), text); + for (const path of migrations) { + write(join(root, 'packages/db/drizzle', path), `candidate ${path}\n`); + write(join(root, 'apps/cli/drizzle', path), `candidate ${path}\n`); + } + write( + join(staging, 'metafile-esm.json'), + JSON.stringify({ + outputs: Object.fromEntries( + Object.keys(js).flatMap((path) => [ + [path, {}], + [`${path}.map`, {}], + ]), + ), + }), + ); + write( + join(staging, 'finalized.json'), + JSON.stringify({ + version: 1, + outputs: Object.entries(js).map(([path, text]) => ({ + path, + bytes: Buffer.byteLength(text), + sha256: createHash('sha256').update(text).digest('hex'), + })), + maps: [], + upstreamMaps: 0, + }), + ); + const files = new Map([ + ...Object.entries(js).map(([path, text]) => [`package/${path}`, Buffer.from(text)]), + ...migrations.map((path) => [`package/drizzle/${path}`, Buffer.from(`candidate ${path}\n`)]), + ['package/package.json', Buffer.from(JSON.stringify(published))], + ['package/README.md', Buffer.from('candidate readme\n')], + ['package/CHANGELOG.md', Buffer.from('candidate changes\n')], + ['package/LICENSE', Buffer.from('candidate license\n')], + ]); + const tarball = gzipSync(ustar(files)); + write(join(artifact, 'relavium-0.1.1.tgz'), tarball); + return await body({ root, staging, artifact, tarball, files, published }); + } finally { + rmSync(temporary, { recursive: true, force: true }); + } +} + +const receiptPath = (artifact) => join(artifact, 'package-test-receipt.json'); + +test('the seal binds a locally produced candidate it is given, never the repository holding seal.mjs', async () => { + await withCandidate(async ({ root, staging, artifact, tarball, files, published }) => { + const receipt = await withOrigin({}, () => + sealPackage(artifact, { repository: root, staging, source: SOURCE, dirty: true }), + ); + assert.equal(receipt.mode, 'compiled_package_test'); + assert.equal(receipt.source, SOURCE); + assert.equal(receipt.dirty, true); + assert.equal(receipt.packageVersion, '0.1.1'); + assert.equal(receipt.tarball, 'relavium-0.1.1.tgz'); + assert.equal(receipt.tarballSha256, createHash('sha256').update(tarball).digest('hex')); + assert.equal(receipt.archiveBytes, tarball.length); + assert.deepEqual(receipt.files.map((file) => file.path).sort(), [...files.keys()].sort()); + // The catalog dependency resolved from the candidate's own workspace; prepack removed. + assert.deepEqual(receipt.package, published); + assert.deepEqual(receipt.origin, { kind: 'local' }); + assert.deepEqual(receipt.toolchain, { node: process.version, pnpm: '9.12.3' }); + assert.deepEqual(JSON.parse(readFileSync(receiptPath(artifact), 'utf8')), receipt); + assert.deepEqual(readdirSync(artifact).sort(), [ + 'package-test-receipt.json', + 'relavium-0.1.1.tgz', + ]); + }); +}); + +test('the seal records the trusted job origin and refuses a dirty source there', async () => { + await withCandidate(async ({ root, staging, artifact }) => { + await withOrigin(GITHUB_ORIGIN, async () => { + await assert.rejects( + sealPackage(artifact, { repository: root, staging, source: SOURCE, dirty: true }), + /DIRTY_SOURCE/, + ); + assert.equal(existsSync(receiptPath(artifact)), false); + const receipt = await sealPackage(artifact, { + repository: root, + staging, + source: SOURCE, + dirty: false, + }); + assert.deepEqual(receipt.origin, { + kind: 'github', + repository: 'owner/name', + workflow: 'CI', + runId: '42', + runAttempt: '1', + event: 'push', + source: SOURCE, + }); + }); + }); +}); + +test('the seal refuses a candidate that pins a different pnpm, writing no receipt', async () => { + await withCandidate( + async ({ root, staging, artifact }) => { + await withOrigin({}, () => + assert.rejects( + sealPackage(artifact, { repository: root, staging, source: SOURCE, dirty: false }), + /pnpm@9\.12\.4/, + ), + ); + assert.equal(existsSync(receiptPath(artifact)), false); + }, + { packageManager: 'pnpm@9.12.4' }, + ); +}); + +test('the seal refuses an invalid source identity before it reads the candidate', async () => { + await withCandidate(async ({ staging, artifact }) => { + const missing = join(artifact, 'no-candidate'); + for (const identity of [ + { source: 'A'.repeat(40), dirty: false }, + { source: SOURCE.slice(1), dirty: false }, + { source: SOURCE, dirty: 'false' }, + ]) + await assert.rejects( + sealPackage(artifact, { repository: missing, staging, ...identity }), + /PACKED_CLI_SOURCE_IDENTITY/, + ); + assert.equal(existsSync(receiptPath(artifact)), false); + }); +}); + +test('the seal refuses archive bytes that differ from the finalized candidate', async () => { + await withCandidate(async ({ root, staging, artifact, files }) => { + files.set('package/LICENSE', Buffer.from('repository license\n')); + writeFileSync(join(artifact, 'relavium-0.1.1.tgz'), gzipSync(ustar(files))); + await withOrigin({}, () => + assert.rejects( + sealPackage(artifact, { repository: root, staging, source: SOURCE, dirty: false }), + /FILE_DRIFT/, + ), + ); + assert.equal(existsSync(receiptPath(artifact)), false); + }); +}); diff --git a/tools/cli-package/smoke/README.md b/tools/cli-package/smoke/README.md index 89d68f26..347113bf 100644 --- a/tools/cli-package/smoke/README.md +++ b/tools/cli-package/smoke/README.md @@ -48,7 +48,8 @@ aggregate. Each result has `id`, `passed`, and evidence or an error. Overall the fixed extensionless fixture script, which creates only its recorded child. The fixture's processes also end once the smoke worker that owns their case is gone, so a consumer interrupted mid-case leaves no orphaned target; while the - worker lives, only the product's own cleanup stops them. Inject a synthetic error on the real target after readiness, then observe actual + worker lives, their owner watch never ends them: the product's own cleanup does, or + on failure the smoke's watchdog and exact-PID cleanup, which remain failure evidence. Inject a synthetic error on the real target after readiness, then observe actual target and descendant closure. Delay delivery of real close events to distinguish an error/request from the owned native acknowledgement. POSIX additionally covers natural leader exit while a descendant holds inherited pipes. Windows runs the diff --git a/tools/cli-package/smoke/index.mjs b/tools/cli-package/smoke/index.mjs index 42eb2a90..e609981b 100644 --- a/tools/cli-package/smoke/index.mjs +++ b/tools/cli-package/smoke/index.mjs @@ -1,5 +1,5 @@ import assert from 'node:assert/strict'; -import { readdirSync, rmSync } from 'node:fs'; +import { mkdtempSync, readdirSync, rmSync } from 'node:fs'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { readOrdinaryFile } from '../files.mjs'; @@ -8,6 +8,15 @@ import { isolatedEnvironment, runCommand } from '../process.mjs'; /** The prefix of the worker's own temporary root under the support directory. */ export const INSTALLED_SMOKE_ROOT_PREFIX = 'installed-smoke-'; +/** + * Create the worker's own temporary root under `supportRoot`, named so that + * `abandonInstalledSmoke` finds it. + * @param {string} supportRoot + */ +export function createInstalledSmokeRoot(supportRoot) { + return mkdtempSync(join(supportRoot, INSTALLED_SMOKE_ROOT_PREFIX)); +} + /** * For a caller that an interrupt ends while the worker runs: the worker is killed before its own * removal, so remove every worker root under `supportRoot`. The native fixture targets the installed diff --git a/tools/cli-package/smoke/native-process.mjs b/tools/cli-package/smoke/native-process.mjs index 5eec4ec0..6a79248f 100644 --- a/tools/cli-package/smoke/native-process.mjs +++ b/tools/cli-package/smoke/native-process.mjs @@ -290,6 +290,22 @@ export function assertNativeBootstrapDiagnostic(diagnostics, mode) { ); } +/** + * The private configuration beside a fixture target's `status` script. Its owner is this smoke + * worker: the fixture's processes end once it is gone (an interrupted consumer). + * @param {{mode: string, log: string, release?: string}} options + */ +export function nativeFixtureConfig({ mode, log, release }) { + return { + mode, + log, + node: process.execPath, + descendant: fixture('native-descendant.cjs'), + owner: process.pid, + ...(release === undefined ? {} : { release }), + }; +} + export async function assertInstalledNativeProcesses(context, { bootstrap = false } = {}) { const { root, cli, environment, calls } = context; const home = join(root, bootstrap ? 'bootstrap-home' : 'native-home'); @@ -339,15 +355,7 @@ export async function assertInstalledNativeProcesses(context, { bootstrap = fals copyFileSync(fixture('native-status.cjs'), join(cwd, 'status')); writeFileSync( join(cwd, 'native-fixture.json'), - JSON.stringify({ - mode, - log, - node: process.execPath, - descendant: fixture('native-descendant.cjs'), - // This worker: the fixture's processes end once it is gone (an interrupted consumer). - owner: process.pid, - ...(bootstrap ? { release } : {}), - }), + JSON.stringify(nativeFixtureConfig({ mode, log, ...(bootstrap ? { release } : {}) })), { mode: 0o600 }, ); writeFileSync( diff --git a/tools/cli-package/smoke/native-process.test.mjs b/tools/cli-package/smoke/native-process.test.mjs index fac06229..e33f5f7c 100644 --- a/tools/cli-package/smoke/native-process.test.mjs +++ b/tools/cli-package/smoke/native-process.test.mjs @@ -22,6 +22,7 @@ import { copyFixtureRuntimeSupport, createFixtureSentinel, InstalledNativeControlError, + nativeFixtureConfig, pendingNativeFixturePids, } from './native-process.mjs'; import { explicitEnvironment, ndjson, poll } from './process.mjs'; @@ -354,13 +355,7 @@ function ownedFixture(t, mode) { ); writeFileSync( join(root, 'native-fixture.json'), - JSON.stringify({ - mode, - log, - node: process.execPath, - descendant: fileURLToPath(new URL('./native-descendant.cjs', import.meta.url)), - owner: owner.pid, - }), + JSON.stringify({ ...nativeFixtureConfig({ mode, log }), owner: owner.pid }), ); const target = spawn(process.execPath, [join(root, 'status')], { cwd: root, @@ -429,6 +424,47 @@ test( }, ); +test('a fixture target is owned by the smoke worker that writes its configuration', () => { + const descendant = fileURLToPath(new URL('./native-descendant.cjs', import.meta.url)); + const common = { log: '/case/native.ndjson', node: process.execPath, descendant }; + assert.deepEqual(nativeFixtureConfig({ mode: 'exit-fallback', log: common.log }), { + mode: 'exit-fallback', + ...common, + owner: process.pid, + }); + assert.deepEqual( + nativeFixtureConfig({ mode: 'bootstrap-fatal', log: common.log, release: '/case/release' }), + { mode: 'bootstrap-fatal', ...common, owner: process.pid, release: '/case/release' }, + ); +}); + +test('a fixture target without an owner refuses at once instead of running unwatched', async (t) => { + const root = mkdtempSync(join(tmpdir(), 'relavium-status-unowned-test-')); + const log = join(root, 'native.ndjson'); + copyFileSync( + fileURLToPath(new URL('./native-status.cjs', import.meta.url)), + join(root, 'status'), + ); + const unowned = nativeFixtureConfig({ mode: 'exit-fallback', log }); + delete unowned.owner; + writeFileSync(join(root, 'native-fixture.json'), JSON.stringify(unowned)); + const target = spawn(process.execPath, [join(root, 'status')], { + cwd: root, + stdio: ['ignore', 'ignore', 'ignore'], + }); + t.after(() => { + if (target.exitCode === null && target.signalCode === null) target.kill('SIGKILL'); + rmSync(root, { recursive: true, force: true }); + }); + const outcome = await Promise.race([ + new Promise((resolve) => target.once('exit', (code) => resolve(code))), + sleep(5000, 'still running', { ref: false }), + ]); + assert.notEqual(outcome, 'still running'); + assert.notEqual(outcome, 0); + assert.equal(existsSync(log), false); +}); + test('shared fixture runtime support reuses read-only identical bytes and refuses replacement content or links', () => { const root = mkdtempSync(join(tmpdir(), 'relavium-shared-runtime-test-')); const source = join(root, 'source.dylib'); diff --git a/tools/cli-package/smoke/worker.mjs b/tools/cli-package/smoke/worker.mjs index ad677e04..5e7295a3 100644 --- a/tools/cli-package/smoke/worker.mjs +++ b/tools/cli-package/smoke/worker.mjs @@ -4,7 +4,6 @@ import { createHash } from 'node:crypto'; import { existsSync, mkdirSync, - mkdtempSync, readFileSync, readdirSync, realpathSync, @@ -21,7 +20,7 @@ import { assertExit, explicitEnvironment, invoke, ndjson } from './process.mjs'; import { assertInstalledNativeProcesses, InstalledNativeControlError } from './native-process.mjs'; import { emitInstalledWorkerFailure } from './failure-diagnostic.mjs'; import { installedModulesOwner } from './installation.mjs'; -import { INSTALLED_SMOKE_ROOT_PREFIX } from './index.mjs'; +import { createInstalledSmokeRoot } from './index.mjs'; const directory = dirname(fileURLToPath(import.meta.url)); const repository = resolve(directory, '../../..'); @@ -64,7 +63,7 @@ async function exerciseInstalledSmoke({ packageRoot, supportRoot, artifactSha256 const before = snapshot(packageRoot); const installedModules = installedModulesOwner(packageRoot); assert.ok(installedModules, 'Installed package must have an explicit node_modules owner'); - const root = mkdtempSync(join(supportRoot, INSTALLED_SMOKE_ROOT_PREFIX)); + const root = createInstalledSmokeRoot(supportRoot); const log = join(root, 'guard.ndjson'); const report = { schemaVersion: 1, diff --git a/tools/cli-package/ustar-fixture.mjs b/tools/cli-package/ustar-fixture.mjs new file mode 100644 index 00000000..9e7974c6 --- /dev/null +++ b/tools/cli-package/ustar-fixture.mjs @@ -0,0 +1,42 @@ +/** + * Test-only USTAR writer: the exact archive shape the inspector accepts (pnpm's layout), so a test + * can build a package tarball from an in-memory inventory. `package/dist/index.js` is 0755, every + * other entry 0644. + */ +import { Buffer } from 'node:buffer'; + +/** @param {Buffer} header */ +export function checksum(header) { + header.fill(32, 148, 156); + header.write( + header + .reduce((sum, byte) => sum + byte, 0) + .toString(8) + .padStart(6, '0'), + 148, + ); + header[154] = 0; + header[155] = 32; +} + +/** @param {string} path @param {Uint8Array} bytes */ +export function entry(path, bytes) { + const header = Buffer.alloc(512); + header.write(path, 0, 100); + for (const offset of [100, 108, 116, 329, 337]) header.write('000000 \0', offset); + header.write(path === 'package/dist/index.js' ? '000755' : '000644', 100); + header.write(bytes.length.toString(8).padStart(11, '0') + ' ', 124); + header.write('00000000000 ', 136); + header[156] = 48; + header.write('ustar\0' + '00', 257); + checksum(header); + return Buffer.concat([header, bytes, Buffer.alloc((512 - (bytes.length % 512)) % 512)]); +} + +/** @param {Map} files */ +export function ustar(files) { + return Buffer.concat([ + ...[...files].map(([path, bytes]) => entry(path, bytes)), + Buffer.alloc(1024), + ]); +} From b0579580d46ba8ca09711c89a5666c5224499afa Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sat, 24 Oct 2026 04:41:54 +0300 Subject: [PATCH 36/97] docs(repo): record the Step 9a.1 acceptance Round 13, a new independent complete cumulative review of 4c3cafe9..cf97b7f1, reported no blocker, high or medium finding and no regression. Its separate security sitting was clean. Under the maintainer's 2026-10-23 acceptance bar, Step 9a.1 is accepted at cf97b7f1. - The register adds round 13 and the acceptance record, and the 9a.1 row links it. CLAUDE.md, AGENTS.md and current.md say 9a.1 is accepted and 9a.2 is next. - deferred-tasks.md records the round's ten verified findings (six distinct). The two test gaps stay open: forwardInterrupts' group forwarding and the map-source bound's refused side. So does the code half of the guard's exit-code nit. - The documentation-only findings are corrected: - packaging.md's store-discovery list names NPM_CONFIG_STORE_DIR; - the deadline's group kill happens when the grace ends or as soon as the stage exits; - the guard's wording says it sets its exit code to 1 before its rule runs; - the freshness lines are current. - The tooling half of the PR #95 nits item is accepted with 9a.1. The duplicate headings stay with 19b. No code changes. Gates: require-packed-cli, release-guard, require-w8-acceptance and workspace-scheduling tests, lint:tools, lint:test-isolation and prettier over the changed files. Refs: ADR-0111, ADR-0051 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 3 +- AGENTS.md | 4 +-- CLAUDE.md | 4 +-- docs/reference/cli/README.md | 2 +- docs/reference/cli/packaging.md | 10 +++--- docs/roadmap/current.md | 8 ++--- docs/roadmap/deferred-tasks.md | 36 +++++++++++++++++-- .../phase-2.5.5-hardening-and-remediation.md | 31 +++++++++++----- tools/cli-package/README.md | 5 +-- tools/cli-package/release-guard.test.mjs | 2 +- 10 files changed, 76 insertions(+), 29 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8eafc2a3..876e307f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -249,7 +249,8 @@ jobs: # Interim fail-closed guard (Wave 3 Step 9a.1, the Interim release guard ruling of 2026-10-19): only a # verified release candidate may publish, and none can exist before Step 9d's custody graph and Wave 4b's # production recipient. Every artifact, including this package-test tarball, therefore stops here; the - # entry point exits 1 before it evaluates anything, so no way of invoking it can skip the refusal. + # entry point sets its exit code to 1 before it evaluates anything, so a rule that returns, throws or + # cannot load still refuses, and no way of invoking it can skip the refusal. - name: Refuse anything but a verified release candidate run: node tools/cli-package/release-guard.mjs ./artifact # Publish the EXACT artifact the smoke matrix proved — not a fresh build. catalog:/workspace: are already diff --git a/AGENTS.md b/AGENTS.md index 0ec1e1a1..fcf4e677 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,7 +117,7 @@ explicit; source CI does not certify later documentation commits. The joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now -under review below; Steps 9a.2–19 have not started) and whole-wave acceptance remains Step 19. +accepted on `development`, below; Steps 9a.2–19 have not started) and whole-wave acceptance remains Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification @@ -130,7 +130,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-implementation-checkpoint--2026-10-22)). `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is next. `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. See [docs/roadmap/current.md](docs/roadmap/current.md) for live status. diff --git a/CLAUDE.md b/CLAUDE.md index 9d84995f..ec89c3af 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -206,7 +206,7 @@ explicit; source CI does not certify later documentation commits. The joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now -under review below; Steps 9a.2–19 have not started) and whole-wave acceptance remains Step 19. +accepted on `development`, below; Steps 9a.2–19 have not started) and whole-wave acceptance remains Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification @@ -219,7 +219,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-implementation-checkpoint--2026-10-22)). `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is next. `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. For live status, per-PR history, milestone dates, and open obligations, see the canonical diff --git a/docs/reference/cli/README.md b/docs/reference/cli/README.md index cca4004f..220a1e7e 100644 --- a/docs/reference/cli/README.md +++ b/docs/reference/cli/README.md @@ -13,5 +13,5 @@ Part of [reference/](../README.md). | [commands.md](commands.md) | The full `relavium` command surface (run, chat, chat-resume/list/export, agent run, list, create, import, export, logs, gate, init, provider) plus interactive TUI vs CI JSON mode. | | [home.md](home.md) | The bare-invocation interactive **Home** — the TTY gate, the read-only management strip over `history.db`, the Home↔chat mode machine + keymap, bracketed paste, the min-terminal degrade, and the signal/exit-code lifecycle (2.5.B / ADR-0054). | | [chat-session.md](chat-session.md) | The `relavium chat` agent-session REPL — entry, agent/model selection, the multi-turn loop, streaming, tool availability, the `--json` session-event stream, and exit code 4. | -| [packaging.md](packaging.md) | Wave 3 finite runtime graph, installed-package consumers and Step 8 native activation accepted on `development`; the Step 9a.1 candidate build and finalizer are implemented (acceptance pending); the remaining Step 9 map/custody and release gates remain staged. | +| [packaging.md](packaging.md) | Wave 3 finite runtime graph, installed-package consumers and Step 8 native activation accepted on `development`; the Step 9a.1 candidate build and finalizer are accepted on `development`; the remaining Step 9 map/custody and release gates remain staged. | | [agent-run-fixture.md](agent-run-fixture.md) | The `relavium agent run --fixture` cassette format — a `StreamChunk[][]` JSON recording of an agent run's LLM stream, for deterministic offline replay. | diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 1d668e2b..6ebcfac7 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -1,6 +1,6 @@ # CLI Packaging Contract -- **Status**: Compiled graph, installed package and Step 8 native activation independently accepted on `development`; the Step 9a.1 candidate build and finalizer are implemented, with independent acceptance pending; the remaining Step 9 map/custody and release gates remain staged +- **Status**: Compiled graph, installed package and Step 8 native activation independently accepted on `development`; the Step 9a.1 candidate build and finalizer are independently accepted on `development` (2026-10-24); the remaining Step 9 map/custody and release gates remain staged - **Date**: 2026-10-15 - **Related**: [ADR-0111](../../decisions/0111-cli-meta-operations-use-a-lazy-packaged-runtime.md), [commands.md](commands.md), [release runbook](../../runbooks/release-a-surface.md) @@ -255,7 +255,7 @@ describes the implemented mechanism; independent acceptance is recorded in the closure is installed there offline, frozen and with `--ignore-scripts` from the existing content-addressed store, which pnpm discovers in the repository with only `HOME`, `USERPROFILE`, `XDG_DATA_HOME`, `XDG_CONFIG_HOME`, `LOCALAPPDATA`, `APPDATA`, - `PNPM_HOME` and `npm_config_store_dir` of the caller's environment. Every `@relavium/*` + `PNPM_HOME`, `npm_config_store_dir` and `NPM_CONFIG_STORE_DIR` of the caller's environment. Every `@relavium/*` link of the CLI and the five inlined engine packages must resolve to that package inside the tree, no other `@relavium/*` link is admitted, and the root `node_modules` must resolve inside the tree too. @@ -314,9 +314,9 @@ starts in their own process groups end on their own once the smoke worker that o gone. On POSIX the local `pnpm smoke:cli-package` forwards an interrupt to the producer or consumer it started rather than ending first (on Windows they share the console and receive Ctrl-C themselves), starts no later stage once one arrived, and exits with the signal's code after -removing its root. On POSIX a stage past its deadline is interrupted (SIGTERM) first and killed, -with its whole process group, only after a grace period, so it stops its own build children before -the root goes; on Windows its whole tree is killed at the deadline, and its private directories go +removing its root. On POSIX a stage past its deadline is interrupted (SIGTERM) first, and its whole +process group is killed when a grace period ends or as soon as the stage exits, whichever comes first, +so it stops its own build children before the root goes; on Windows its whole tree is killed at the deadline, and its private directories go with the check's root. These Windows interrupt and deadline paths are not exercised by any CI job: the tooling tests run on Linux only. SIGHUP aborts and cleans up like the other two signals even under `nohup`, which these listeners replace; detach a long local run with `setsid` or a terminal diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index 7edacc35..ba8c74da 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -1,7 +1,7 @@ # Current state > Status: Living -> Last updated: 2026-10-22 +> Last updated: 2026-10-24 - **Related**: [README.md](README.md), [phases/phase-2.5-cli-consolidation.md](phases/phase-2.5-cli-consolidation.md), [phases/phase-2.5.5-hardening-and-remediation.md](phases/phase-2.5.5-hardening-and-remediation.md), [phases/phase-2-cli.md](phases/phase-2-cli.md), [deferred-tasks.md](deferred-tasks.md), [../project-structure.md](../project-structure.md), [../tech-stack.md](../tech-stack.md) @@ -138,7 +138,7 @@ The [deferred register](deferred-tasks.md) retains its existing limits; closure **Wave 3 Steps 1–8 merged to `main` on 2026-10-18 as [PR #95](https://github.com/HodeTech/Relavium/pull/95), merge commit `6d2a7372` (22:44 UTC). Wave 3 itself is not complete: Steps 9–19 had not started at -that merge (Step 9a.1 is now under review; Steps 9a.2–19 have not started) and whole-wave acceptance +that merge (Step 9a.1 is now accepted on `development`; Steps 9a.2–19 have not started) and whole-wave acceptance remains Step 19.** The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) joins the completed scope, every review attribution, the independent review's dispositions and the publication state observed at merge. It was the maintainer-authorized, scoped publication checkpoint. @@ -172,7 +172,7 @@ bar and round cap, as its medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, by the author alone). Its other findings are recorded in -[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is implemented on `development` and under review ([checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-implementation-checkpoint--2026-10-22)); the substrate's +[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is independently accepted on `development` at `cf97b7f1` (2026-10-24): its round 13 reported no blocker, high or medium finding and its security sitting was clean, and the round's low findings are recorded in deferred-tasks.md ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)); Step 9a.2 is next; the substrate's 9a.1–9d increments carry a dedicated archive/crypto security sitting. Steps 10–19 follow the [increment table](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) and end with whole-wave acceptance at Step 19. `development` is pushed after each accepted increment; PR #95 no longer bounds the @@ -1141,7 +1141,7 @@ an ESLint selector rather than a list. ### Wave 3 — Clear the ground -**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Step 9a.1 implemented on `development` and under review, Steps 9a.2–19 NOT STARTED — 2026-10-22.** +**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Step 9a.1 accepted on `development` (2026-10-24), Steps 9a.2–19 NOT STARTED — 2026-10-24.** ADR-0110–0113, their linked contracts and the nineteen-increment plan were approved on 2026-10-16. [Steps 1–8 were independently accepted](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) on `development`, with whole Step 8 reviewed at `c83c8575`, and published as the scoped checkpoint diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index ae1e7d47..5930ea2d 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -2,7 +2,7 @@ > Status: Living -> Last updated: 2026-10-22 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Step 9a.1 is implemented on `development` and under review, Steps 9a.2–19 not started; the PR95 increment is closed on `development` under the maintainer's 2026-10-21 round cap). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's edge cases with the maintainer's 2026-10-21 acceptance bar and round cap and the 2026-10-22 rulings. Existing approved deferrals remain. +> Last updated: 2026-10-24 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Step 9a.1 is independently accepted on `development` with its recorded low findings below, Steps 9a.2–19 not started; the PR95 increment is closed on `development` under the maintainer's 2026-10-21 round cap). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's edge cases with the maintainer's 2026-10-21 acceptance bar and round cap and the 2026-10-22 rulings. Existing approved deferrals remain. > The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) records completed prerequisites; full M/N lifecycle and other scheduled work remain with their owners. > > Earlier triage: the Phase-2.6 rewrite triaged every open item, the 2026-07-19 full-project @@ -434,7 +434,7 @@ measured or counted here. `apps/cli/src/harness/concurrency.e2e.test.ts:434-441` uses a fixed 100 ms contention margin; `tools/cli-package/consumer-lifetime.test.mjs:54-59` pre-marks every synthetic case `passed: true` (by design; real detection is the packed-consumer CI). *(nit · the files above)* -- [ ] **Scheduled → Wave 3: the two tooling nits → 9a.1; the duplicate headings → 19b. Tooling and documentation nits.** *Implemented in Wave 3 Step 9a.1 on 2026-10-22, acceptance pending:* the unreachable `TAR_FORMAT` branches are removed, and `installedModulesOwner` (`tools/cli-package/smoke/installation.mjs`) replaces the ancestor loop; the duplicate headings stay with 19b. *Nit, (reviewer).* *Premise corrected 2026-10-19:* the repeated headings are the 2.5.5.A–I sub-stream headings and the Wave 2/Wave 3 "Decisions presented together before implementation" pair, not Wave 2 checkpoint sub-headings. Unreachable `TAR_FORMAT` branches at +- [ ] **Scheduled → Wave 3: the two tooling nits → 9a.1; the duplicate headings → 19b. Tooling and documentation nits.** *The tooling half is accepted with Wave 3 Step 9a.1 (implemented 2026-10-22, accepted at `cf97b7f1` on 2026-10-24):* the unreachable `TAR_FORMAT` branches are removed, and `installedModulesOwner` (`tools/cli-package/smoke/installation.mjs`) replaces the ancestor loop; the duplicate headings stay with 19b. *Nit, (reviewer).* *Premise corrected 2026-10-19:* the repeated headings are the 2.5.5.A–I sub-stream headings and the Wave 2/Wave 3 "Decisions presented together before implementation" pair, not Wave 2 checkpoint sub-headings. Unreachable `TAR_FORMAT` branches at `tools/cli-package/archive.mjs:126,135`; no `break` in the ancestor loop at `tools/cli-package/smoke/worker.mjs:64-65` (the outermost `node_modules` wins); and repeated heading texts in the phase register, whose later occurrences take a `-1` slug suffix. *(author)* A scan finds repeated H2–H4 headings (the 2.5.5.B–I sub-stream headings and the @@ -616,6 +616,38 @@ real run that shows the shape. once with `graceMs > 0`) and of `tools/bundle-closure/closure.test.mjs`. *(low · tools/cli-package/process.mjs, tools/cli-bootstrap/process.mjs, tools/bundle-closure/inventory.mjs)* +Round 13, the acceptance round at `cf97b7f1` (no blocker, high or medium finding; security sitting clean), +reported ten verified findings, six of them distinct. Under the 2026-10-23 bar they are recorded here; the +documentation-only ones were also corrected in the acceptance commit, which changes no code. + +- [ ] **`forwardInterrupts`' group-wide forwarding has no test.** *Low.* *(round 13, TCA-9A1-R13-1)* + `forwardInterrupts` sends the signal to each child's process group (`process.kill(-child.pid, signal)`), as + its docblock says, but changing it to `process.kill(child.pid, signal)` passes `process.test.mjs` and + `check.test.mjs`: the forwarding test checks only the leader's own marker. Every stage child today is + detached into its own group, so the real composition behaves the same. **Re-check:** in the forwarding + harness, have the detached child start a non-detached grandchild in its group that writes a marker on + SIGINT, and assert both markers. *(low · tools/cli-package/process.mjs)* +- [ ] **The map-source length bound is pinned on its admitted side only.** *Nit.* *(round 13, TCA-9A1-R13-2)* + `finalize.test.mjs` admits a 1024-character source and refuses a 1031-character one, so loosening + `source.length > 1024` to any bound up to 1030 passes. The source must still resolve to an existing original + file, so nothing is exposed. **Re-check:** add a 1025-character source (`../src/${'a/'.repeat(507)}x.ts`) + expecting `MAP_SOURCE_PATH`. *(nit · tools/cli-package/finalize.test.mjs)* +- [ ] **A rule that lowers the exit code would let the guard exit 0.** *Nit, code half.* *(round 13, + CRIT-9A1-R13-N1)* `release-guard.mjs` sets `process.exitCode = 1` once, so a rule that set + `process.exitCode = 0` or called `process.exit(0)` would make the publish guard exit 0; today's rule always + throws, and the guarantee that no way of invoking the guard skips the refusal holds. The wording that said + the entry point "exits 1" is corrected. **Re-check:** when Step 9d replaces the rule, re-assert the code before + the final line and keep it with an `exit` listener, with a test for each variant. + *(nit · tools/cli-package/release-guard.mjs)* +- [x] **Documentation corrected in the acceptance commit.** *(round 13)* The contract's store-discovery list + now names `NPM_CONFIG_STORE_DIR`, which the code forwards (low; PLAN-9A1-R13-1 = INT-9A1-R13-1, also + PFC-R13-N1 and TCA-9A1-R13-3); the deadline's group kill is described as happening when the grace ends or as + soon as the stage exits (PLAN-9A1-R13-N1); the guard's wording says it sets its exit code to 1 before its rule + runs (CRIT-9A1-R13-N1, wording half); and this file's freshness line is current (PLAN-9A1-R13-N2 = + INT-9A1-R13-N1). **Re-check:** the contract's list equals `STORE_DISCOVERY_VARIABLES` in + `tools/cli-package/candidate.mjs`. *(docs/reference/cli/packaging.md, tools/cli-package/README.md, + .github/workflows/release.yml)* + ## Phase 2.6.5 deferrals — `CR-93` and `CR-95` (long half), recorded 2026-09-06; `CR-70`, `CR-94` (remainder) and the in-turn overflow continuation, recorded 2026-09-14; the errored-or-aborted turn residual, the pre-upgrade free-page bytes and the estimator's calibration, recorded 2026-09-18 **Phase closure — 2026-10-11:** W8 merged to `main` in PR #92 and Phase 2.6.5 is complete diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 0eb96bc0..0b8fd64d 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -3697,7 +3697,7 @@ No credential values or request captures belong in tracked evidence. | PR95 | **Recorded plan amendment, 2026-10-19 (maintainer-approved; new scope from the [PR #95 independent review](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18), not a split of rows 1–19) — scrubber residuals, before 9a:** the `api-key:Bearer`/`Basic` label erasure that leaks the token; raw `?`/`#` URL userinfo masked unless the text before it is host-shaped **and** an `=` assignment separates it from the authority `@`, applied to the LLM leaf and core's twin predicate; an extended finite scheme list (postgres, postgresql, mysql, mariadb, redis, rediss, mongodb, mongodb+srv, amqp, amqps, ssh, sftp, ldap, ldaps, smtp, smtps) masking colon-bearing userinfo for the new schemes; module-load checks of the split-credential scanner's two invariants, a start prefilter, joined-view and glued-scheme witnesses, the load-sensitive 8 MiB timing case, and the causal-control pins. One increment with ordered commits, two NEW clean complete cumulative rounds and a separate security sitting. *Acceptance amended on 2026-10-21 by the maintainer's masker acceptance bar and round cap; closed under the round cap on 2026-10-22 ([checkpoint](#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)).* | | 9 | **Release map/custody substrate:** #46/ADR-0111, independently reviewed after the finite graph. Upstream TS maps, sole prepack final build, pre-hash trailer removal/staging, exact tarball/CLI+engine symbolication, encrypt-once candidate with permissioned prebuild discovery/reservation → pack/reuse → smoke → archive → npm → finalization. Required artifact/platform aggregation, all rerun modes, lost-staging refusal and permission-enforcing offline API controls. Dedicated archive/crypto security sitting. Fixture proof never claims production recipient/custody configuration. | | 9a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage #46/ADR-0111 unchanged, no new contract) — release maps, final build and symbolication:** fresh uncached upstream original-TypeScript maps, the sole prepack final CLI build, final trailer removal before hashes, map and private-graph staging outside public artifacts and caches, exact tarball-to-final-JS binding, and real packed CLI and engine frame symbolication with wrong JS/map/source and missing-map negative controls. Ordinary builds remain map-free. *Split again on 2026-10-19 into 9a.1 and 9a.2 below, which replace it as increments ([Plan shape](#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19)).* | -| 9a.1 | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — candidate build and finalizer:** an owned isolated candidate copy, fresh uncached upstream maps, candidate-mode tsup selection with absent `sourcesContent`, the prepack finalizer (upstream and final map validation, repository-relative sources, exact trailer removal before hashes), owner-only private staging outside public artifacts and caches, closure/seal adaptation, the ordinary map-free control, the PR #95 archive/worker fold-ins, and a fail-closed `release.yml` publish guard that refuses any non-release candidate until 9d and Wave 4b custody exist. The receipt stays `compiled_package_test`. *Implemented on `development` on 2026-10-22; acceptance pending ([checkpoint](#wave-3-step-9a1-implementation-checkpoint--2026-10-22)).* | +| 9a.1 | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — candidate build and finalizer:** an owned isolated candidate copy, fresh uncached upstream maps, candidate-mode tsup selection with absent `sourcesContent`, the prepack finalizer (upstream and final map validation, repository-relative sources, exact trailer removal before hashes), owner-only private staging outside public artifacts and caches, closure/seal adaptation, the ordinary map-free control, the PR #95 archive/worker fold-ins, and a fail-closed `release.yml` publish guard that refuses any non-release candidate until 9d and Wave 4b custody exist. The receipt stays `compiled_package_test`. *Implemented on `development` on 2026-10-22 ([checkpoint](#wave-3-step-9a1-implementation-checkpoint--2026-10-22)); independently accepted at `cf97b7f1` on 2026-10-24 ([acceptance](#wave-3-step-9a1-acceptance--2026-10-24)).* | | 9a.2 | **Recorded split — manifest, symbolication and package-test receipt:** the private map manifest keyed by `dist/.js`, actual packed CLI and engine frame symbolication against the matching source revision, wrong JS/map/source and missing/malformed-upstream negative controls (product copies via `check:map-controls`, also a required uncached CI job), and the atomic `package_test` receipt flip across seal, binding, consumers, aggregate and both workflows. | | 9b | **Recorded split — recipient custody:** synthetic-only isolated GnuPG controls (public-only home/keyring, pinned primary and encryption-subkey fingerprints, machine-readable refusal of missing, expired, revoked or unusable recipients), encrypt once and reuse the original ciphertext. Explicit crypto/custody security sitting. Fixture proof never claims production recipient or custody configuration. | | 9c | **Recorded split — offline release state controller:** permission-enforcing offline GitHub/npm controller, permissioned prebuild discovery and reservation, and every rerun, partial, conflict and expiry state, with no overwrite, rebuild or re-encryption on uncertain history. | @@ -9596,6 +9596,7 @@ adversarially verified; the detailed reports stay local and Git-ignored. | 10 | `954d6a62` | NONCLEAN (plan, correctness, integration and critic clean) | 4 (3 low, 1 nit) | with a corepack-managed pnpm, the producer's pnpm ran from the isolated build home under the shared temporary directory, so its optional module lookups reached a `node_modules` there that another local user could create (present since Step 7b; shared-`/tmp` POSIX hosts only, never CI): isolated children now carry the developer's own corepack cache (`COREPACK_HOME`, computed as corepack computes it), and a test plants such a module and shows it never runs. A failed removal during an interrupt escaped the signal handler with a raw stack and exit 1: each cleanup step is now attempted on its own, a failure leaves one fixed line, the exit code stays the signal's, and an ordinary failed removal no longer skips the other directory. A source root's sibling sharing its prefix gained a killing test, and an untracked nested repository refuses under its own code | | 11 | `d890abf0` | NONCLEAN (plan, security, correctness and integration lenses clean) | 4 (1 medium, 3 low) | nothing pinned that the producer looks up the developer's global excludes in the caller's own environment rather than the isolated build environment, which would have dropped them silently: the composition test now requires the very environment object. The local check's deadline killed only a stage's own process group, so a producer's detached build child could recreate the removed root with a map: a stage past its deadline is now interrupted first (`runCommand`'s `graceMs`) and killed only after a grace period, tested with a late-writing build child. The check's wait for an interrupt caught during its own root removal gained a killing test, and `dependencyStore`'s docblock says where corepack runs | | 12 | `78623964` | NONCLEAN (plan, security and correctness lenses clean) | 7 (1 medium, 2 low, 4 nits; 5 distinct) | Git reading the checkout got Node's null device, `\\.\nul` on Windows, which Git for Windows refuses (it maps only `nul` and the literal `/dev/null`), so every isolated Git call would exit on a Windows host — the local producer and a local consumer; the CI producer runs on Linux and every CI consumer names its source: it now gets the literal `/dev/null`. The consumer stage's deadline grace and a bounded grace test gained killing tests, and the store discovery's exception is stated. The author gate statements of rounds 3–11 said the filesystem and secrets families' captured sources were unchanged, but both capture the root `package.json`, which round 3 changed (one acceptance-tools entry); this round runs the full `pnpm run ci`, every family included | +| 13 | `cf97b7f1` | **ACCEPTED** under the 2026-10-23 bar (security sitting, tests and integration lenses clean; no blocker, high or medium finding and no regression) | 10 (3 low, 7 nits; 6 distinct) | none in the loop. The contract's list of the caller variables that reach store discovery omitted `NPM_CONFIG_STORE_DIR`, which the code forwards (low, reported by four lenses), and three wording nits (the deadline's group kill, the guard's exit code, deferred-tasks.md's freshness line) are documentation only and corrected in the acceptance commit; `forwardInterrupts`' group-wide forwarding has no test (low) and the map-source bound is pinned on its admitted side only (nit), both recorded in deferred-tasks.md | **Internal rigor pass, 2026-10-24.** After the twelfth round the maintainer asked for maximum care before any further round, so an internal read-only pass covered the whole increment at `1f247772` before round @@ -9626,11 +9627,23 @@ so every wait for a spawned tool's exit in the interrupt tests is now bounded an test instead. Five low findings that need a change of scope are recorded in [deferred-tasks.md](../deferred-tasks.md#wave-3-step-9a1--recorded-low-findings-2026-10-24). -**Remaining gates.** The increment is not independently accepted. Under the Acceptance row as amended -on 2026-10-23, one NEW complete cumulative round with no blocker, high or medium finding and no -regression remains, together with the separate scoped security sitting the Acceptance row requires, at -the same head and with the same bar (it runs alongside that round as its own reviewer and report, as in -every round so far); their low findings go to deferred-tasks.md. Then `development` is pushed and its CI observed, and the PR #95 -tooling nits in -[deferred-tasks.md](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18) -close. +**Remaining gates.** None for acceptance: see the acceptance record below. `development` is pushed and +its CI observed after the acceptance commit. + +### Wave 3 Step 9a.1 acceptance — 2026-10-24 + +**Step 9a.1 is independently accepted on `development` at `cf97b7f1` (2026-10-24)** under the Acceptance +row as amended on 2026-10-23. Round 13, a NEW independent complete cumulative review of +`4c3cafe9..cf97b7f1` with every reviewer in its own worktree detached at that head, reported no blocker, +high or medium finding and no regression, and its separate security sitting, run at the same head as its +own reviewer and report, was clean. The plan reviewer ran the real producer at the reviewed head: it exited +0 with a `compiled_package_test` receipt, and the tarball held no map, map link or `sourcesContent`. The +round's ten verified findings (three low, seven nits; six distinct) are recorded in +[deferred-tasks.md](../deferred-tasks.md#wave-3-step-9a1--recorded-low-findings-2026-10-24). Those that +concern documentation only — the contract's store-discovery list, the deadline's group-kill wording, the +guard's exit-code wording and deferred-tasks.md's freshness line — are corrected in the acceptance commit, +which changes no code; the two test gaps stay open there. The PR #95 tooling nits owned by 9a.1 (the +unreachable `TAR_FORMAT` branches and the smoke worker's ancestor loop) are accepted with it; that item's +duplicate headings stay with 19b. Author evidence at `cf97b7f1`: the full `pnpm run ci` with every +causal-control family, and the acceptance tools (835 of 835) with and without `CI=true GITHUB_ACTIONS=true`. +The receipt stays `compiled_package_test`; Step 9a.2 is next. diff --git a/tools/cli-package/README.md b/tools/cli-package/README.md index 8a10d530..760a78c1 100644 --- a/tools/cli-package/README.md +++ b/tools/cli-package/README.md @@ -62,8 +62,9 @@ temporary directory that holds the isolated homes (`corepackHome` in `process.mj encrypted custody or a production release candidate (Steps 9a.2–9d). No private map, metafile or per-source inventory enters the public artifact. The release workflow's `release-guard.mjs` refuses every artifact until verified release custody exists: it -exits 1 before it evaluates its rule (`release-guard-core.mjs`), so no way of invoking it -can skip the refusal. +sets its exit code to 1 before it evaluates its rule (`release-guard-core.mjs`), so a rule +that returns, throws or cannot load still refuses, and no way of invoking it can skip the +refusal. The consumer verifies the chosen bytes before installation, installs a copy outside the workspace, and compares every installed public file with the archive receipt. diff --git a/tools/cli-package/release-guard.test.mjs b/tools/cli-package/release-guard.test.mjs index a8dc5989..ea74df17 100644 --- a/tools/cli-package/release-guard.test.mjs +++ b/tools/cli-package/release-guard.test.mjs @@ -117,7 +117,7 @@ for (const [label, rule, stderr] of [ "const error = new Error('x'); error.code = 'ENOENT'; throw error;", '::error::CLI publication refused: GUARD_FAILURE\n', ], - // Whatever the rule does to the process, the exit code was already 1 before it ran. + // The exit code was set to 1 before the rule ran, so a bare exit keeps it. ['exits the process itself', 'process.exit();', ''], ]) { test(`the guard refuses when its rule ${label}`, () => { From 6277e375dcb29114feb0008b37ff70d58677a8de Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sat, 24 Oct 2026 08:00:22 +0300 Subject: [PATCH 37/97] docs(repo): record the Step 9a.2 split and preflight decisions A read-only preflight mapped everything Step 9a.2 touches before any code was written, and the maintainer approved each recommendation on 2026-10-24. - 9a.2 is split into 9a.2a (decoder, finalizer hardening, private manifest, frame capture and symbolication, receipt unchanged) and 9a.2b (check:map-controls, the required uncached CI job, then the atomic package_test flip). The plan now has 30 increments. - The rulings fix how real CLI and core frames are captured (a tooling-only preload during an offline import, on the finalized dist proven equal to the tarball). They add a per-source digest inventory to the private manifest and say how the matching source revision is read. The package-test producer's post-receipt rebuild refusals move into 9a.2b. - The register records 9a.1's push CI: every required job passed. The advisory Windows job failed on a database test that lacks its siblings' explicit timeout. - deferred-tasks.md records that timeout and the inert --verbose stack branch, which Step 10a replaces. No code changes. Refs: ADR-0111 Co-Authored-By: Claude Opus 5.5 --- docs/roadmap/deferred-tasks.md | 19 +++++++++ .../phase-2.5.5-hardening-and-remediation.md | 39 ++++++++++++++++++- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index 5930ea2d..0f8c0d5c 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -648,6 +648,25 @@ documentation-only ones were also corrected in the acceptance commit, which chan `tools/cli-package/candidate.mjs`. *(docs/reference/cli/packaging.md, tools/cli-package/README.md, .github/workflows/release.yml)* +Observed in 9a.1's push CI and its preflight for 9a.2 (2026-10-24): + +- [ ] **A migration-running database test lacks the explicit timeout its siblings carry.** *Low.* *(author, + push CI run 40050981969)* In `packages/db/src/session-effect-turns.test.ts`, the parametrized case + `refuses issuance inside an outer %s transaction before any key can escape a rollback` runs every migration + per case under vitest's default 5 s, while its siblings that migrate pass `30_000`; on the advisory Windows + runner the `exclusive` case took 7.7 s and failed. 9a.1 changed no database source. The advisory Windows job + failed 6 times in the last 40 CI runs, each a default 5 s timeout on a different SQLite- or + process-heavy test (`fs-workspace-admission`, `concurrency.e2e`, the runner-abort ownership case and this + one); the job is advisory and every required job passed. **Re-check:** give the case its siblings' `30_000` + bound, and review the other three tests' bounds against the Windows runner, when the file is next edited or + the advisory job next fails. *(low · packages/db/src/session-effect-turns.test.ts)* +- [ ] **`--verbose` never prints a real stack.** *Low, for Step 10a.* *(9a.2 preflight)* + `apps/cli/src/process/render-user-facing.ts` (`ownErrorStack`) prints a stack only when `stack` is an own + data property, but V8 defines a real error's `stack` as an accessor (get/set), so a thrown `Error` prints + nothing under `--verbose`; `render-error.test.ts` constructs a data `stack`, so the tests pass. Step 10a + replaces this raw-stack branch (row 10a). **Re-check:** the replacement prints a real error's frames, tested + with a thrown `Error` rather than a constructed property. *(low · apps/cli/src/process/render-user-facing.ts)* + ## Phase 2.6.5 deferrals — `CR-93` and `CR-95` (long half), recorded 2026-09-06; `CR-70`, `CR-94` (remainder) and the in-turn overflow continuation, recorded 2026-09-14; the errored-or-aborted turn residual, the pre-upgrade free-page bytes and the estimator's calibration, recorded 2026-09-18 **Phase closure — 2026-10-11:** W8 merged to `main` in PR #92 and Phase 2.6.5 is complete diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 0b8fd64d..8327a873 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -3698,7 +3698,9 @@ No credential values or request captures belong in tracked evidence. | 9 | **Release map/custody substrate:** #46/ADR-0111, independently reviewed after the finite graph. Upstream TS maps, sole prepack final build, pre-hash trailer removal/staging, exact tarball/CLI+engine symbolication, encrypt-once candidate with permissioned prebuild discovery/reservation → pack/reuse → smoke → archive → npm → finalization. Required artifact/platform aggregation, all rerun modes, lost-staging refusal and permission-enforcing offline API controls. Dedicated archive/crypto security sitting. Fixture proof never claims production recipient/custody configuration. | | 9a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage #46/ADR-0111 unchanged, no new contract) — release maps, final build and symbolication:** fresh uncached upstream original-TypeScript maps, the sole prepack final CLI build, final trailer removal before hashes, map and private-graph staging outside public artifacts and caches, exact tarball-to-final-JS binding, and real packed CLI and engine frame symbolication with wrong JS/map/source and missing-map negative controls. Ordinary builds remain map-free. *Split again on 2026-10-19 into 9a.1 and 9a.2 below, which replace it as increments ([Plan shape](#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19)).* | | 9a.1 | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — candidate build and finalizer:** an owned isolated candidate copy, fresh uncached upstream maps, candidate-mode tsup selection with absent `sourcesContent`, the prepack finalizer (upstream and final map validation, repository-relative sources, exact trailer removal before hashes), owner-only private staging outside public artifacts and caches, closure/seal adaptation, the ordinary map-free control, the PR #95 archive/worker fold-ins, and a fail-closed `release.yml` publish guard that refuses any non-release candidate until 9d and Wave 4b custody exist. The receipt stays `compiled_package_test`. *Implemented on `development` on 2026-10-22 ([checkpoint](#wave-3-step-9a1-implementation-checkpoint--2026-10-22)); independently accepted at `cf97b7f1` on 2026-10-24 ([acceptance](#wave-3-step-9a1-acceptance--2026-10-24)).* | -| 9a.2 | **Recorded split — manifest, symbolication and package-test receipt:** the private map manifest keyed by `dist/.js`, actual packed CLI and engine frame symbolication against the matching source revision, wrong JS/map/source and missing/malformed-upstream negative controls (product copies via `check:map-controls`, also a required uncached CI job), and the atomic `package_test` receipt flip across seal, binding, consumers, aggregate and both workflows. | +| 9a.2 | **Recorded split — manifest, symbolication and package-test receipt:** the private map manifest keyed by `dist/.js`, actual packed CLI and engine frame symbolication against the matching source revision, wrong JS/map/source and missing/malformed-upstream negative controls (product copies via `check:map-controls`, also a required uncached CI job), and the atomic `package_test` receipt flip across seal, binding, consumers, aggregate and both workflows. *Split on 2026-10-24 (maintainer-approved; ID coverage #46/ADR-0111 unchanged) into 9a.2a and 9a.2b below, which replace it as increments ([preflight decisions](#wave-3-step-9a2-preflight-decisions--2026-10-24)).* | +| 9a.2a | **Recorded split, 2026-10-24 (maintainer-approved) — decoder, private manifest and symbolication:** an in-house strict source-map decoder; strict mapping and range validation of upstream and final maps in the finalizer; the private map manifest (`staging/manifest.json`, keyed by `dist/.js`, with a per-source digest inventory and the commit's tree), sealed after the archive inspection; real packed CLI and `@relavium/core` frame capture through a tooling-only preload during an offline `relavium import`, run on the finalized `dist` that the inspection proves byte-equal to the tarball; symbolication of those frames against the matching source revision (clean: the checkout's Git objects; dirty, local only: the candidate copy), wired into the seal before the receipt is written. The receipt stays `compiled_package_test`. Security sitting. | +| 9a.2b | **Recorded split, 2026-10-24 (maintainer-approved) — product map controls and the package-test receipt:** `check:map-controls` over product copies (wrong JS by line and column, wrong map, permuted sources, wrong source, missing, malformed and truncated upstream maps, a shifted valid upstream map, and the package-test producer's post-receipt rebuild refusals that ADR-0111 lists), run in `pnpm run ci` and as a required uncached CI job inside `ci`'s needs (the required contexts stay `ci` and `coverage`); then the atomic `package_test` flip across seal, binding, consumers, aggregate and both workflows, version 1 kept. Security sitting. | | 9b | **Recorded split — recipient custody:** synthetic-only isolated GnuPG controls (public-only home/keyring, pinned primary and encryption-subkey fingerprints, machine-readable refusal of missing, expired, revoked or unusable recipients), encrypt once and reuse the original ciphertext. Explicit crypto/custody security sitting. Fixture proof never claims production recipient or custody configuration. | | 9c | **Recorded split — offline release state controller:** permission-enforcing offline GitHub/npm controller, permissioned prebuild discovery and reservation, and every rerun, partial, conflict and expiry state, with no overwrite, rebuild or re-encryption on uncertain history. | | 9d | **Recorded split — workflow, runbook and whole-Step-9 landing:** least-privilege actual workflow dependency graph, required artifact/platform aggregation and exact-artifact consumers, lost-staging refusal and the release runbook update. The whole-Step-9 cumulative review retains all 9a–9d changes with preservation traces into Step 8, and the dedicated archive/crypto security sitting closes the step. | @@ -9395,7 +9397,7 @@ plan amendment (row PR95) and the recorded splits; ID coverage is unchanged apar | --- | --- | | Acceptance | Each increment keeps the existing rule: implement, author gates, commit, then NEW independent complete cumulative review rounds until two consecutive rounds are clean after the last material fix. Comment- or documentation-only nits do not reset the count. Security-touching increments add a separate scoped sitting: PR95, 9a.1, 9a.2, 9b, 9c, 9d (whole Step 9 custody), 10a, 10c, 11b, 13c, 14b, 15c, 16a, 16b, 16c (whole Step 16), 18 and 19b (whole wave). PR95 instead follows the maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)). **Amended 2026-10-23 (maintainer, after Step 9a.1's twelfth round):** from Step 9a.1's round 13 on, every increment is accepted after one NEW independent complete cumulative round that reports no blocker, high or medium finding and no regression; that round's low findings are recorded in [deferred-tasks.md](../deferred-tasks.md) with their file, a minimal example and a re-check note rather than fixed in the loop, and nits are optional. A high or medium finding is still corrected and followed by a new round, and the separate security sittings above are unchanged. | | Publication | `development` is pushed after each accepted increment, so its actual CI (including the four same-tarball platform consumers) is observed per increment. The Wave 3 PR follows Step 19. | -| Plan shape | PR95 + 9a.1, 9a.2, 9b, 9c, 9d + 10a–c + 11a–b + 12a–b + 13a–c + 14a–b + 15a–c + 16a–c + 17a–b + 18 + 19a–b (29 increments). Each split is recorded before its step starts. | +| Plan shape | PR95 + 9a.1, 9a.2, 9b, 9c, 9d + 10a–c + 11a–b + 12a–b + 13a–c + 14a–b + 15a–c + 16a–c + 17a–b + 18 + 19a–b (29 increments). Each split is recorded before its step starts. **Amended 2026-10-24 (maintainer):** 9a.2 is split into 9a.2a and 9a.2b, so the plan has 30 increments ([9a.2 preflight decisions](#wave-3-step-9a2-preflight-decisions--2026-10-24)). | | Reassignments | G54 moves from row 17 to 11a; #311's implementation moves to 10b and its certification stays in 17a; G36/G46 are re-scoped in 10a to the branch that replaces the raw-stack branch and are certified in 17b; promotion of the Steps 6–10 commands.md sections moves from Step 19 to 10c. | | Interim release guard | 9a.1 adds a fail-closed check to the `release.yml` publish job that refuses any non-release candidate. Consequence, acknowledged: no npm publication, including a v0.1.x patch, until 9d and Wave 4b production custody exist. | | PR95 scrubber residuals | Fixed before 9a: the `api-key:Bearer`/`Basic` leak; raw `?`/`#` userinfo masked unless the text before it is host-shaped and an `=` assignment separates it from the authority `@`, in both the LLM leaf and core's twin predicate (core's other documented differences are recorded, not removed); an extended finite scheme list that masks colon-bearing userinfo for the new schemes; the scanner invariants, prefilter and witnesses; the load-sensitive 8 MiB timing case. LOW-1 and LOW-2 become approved deferrals with their recorded triggers. | @@ -9647,3 +9649,36 @@ unreachable `TAR_FORMAT` branches and the smoke worker's ancestor loop) are acce duplicate headings stay with 19b. Author evidence at `cf97b7f1`: the full `pnpm run ci` with every causal-control family, and the acceptance tools (835 of 835) with and without `CI=true GITHUB_ACTIONS=true`. The receipt stays `compiled_package_test`; Step 9a.2 is next. + +**Push CI.** `development` was pushed at `b0579580` (the acceptance commit on top of `cf97b7f1`). Every +required job of [run 40050981969](https://github.com/HodeTech/Relavium/actions/runs/40050981969) passed: the +originating pack, the four same-tarball installed-CLI consumers (current Linux, macOS and Windows and Node +22.0.0 Linux) and their aggregate, lint, typecheck and test, the engine coverage floor and the causal-control +families. The advisory Windows job failed: one `packages/db` test timed out at vitest's default 5 s on that +runner (7.7 s). 9a.1 changed no database or CLI source; the cause and the runner's history are recorded in +[deferred-tasks.md](../deferred-tasks.md#wave-3-step-9a1--recorded-low-findings-2026-10-24). + +### Wave 3 Step 9a.2 preflight decisions — 2026-10-24 + +A read-only preflight (five readers and a synthesis; the detailed record and a Turkish decision register are +local and Git-ignored) mapped every place Step 9a.2 touches before any code was written. The maintainer +approved each recommendation on 2026-10-24; no new ADR or runtime dependency follows. + +| Decision | Ruling | +| --- | --- | +| Split | 9a.2 becomes 9a.2a (decoder, finalizer hardening, private manifest, frame capture and symbolication before the receipt write; the receipt stays `compiled_package_test`) and 9a.2b (`check:map-controls`, the required uncached CI job and its pins, then the atomic `package_test` flip). `package_test` is never claimed before its negative controls exist. | +| Frames | "Actual packed CLI and engine frames" are captured by a tooling-only `node --import` preload while the packed CLI runs an offline `relavium import` of a canary document: the preload wraps the external `yaml` package's `parse` and records one synchronous stack of structured call sites for that document alone, crossing the CLI and `@relavium/core`. Engine coverage is `@relavium/core` (at least two core and three CLI anchors). No product surface is added and no native module is loaded. | +| Where they run | The finalized `apps/cli/dist`, which the archive inspection has proven byte-equal to the tarball entry by entry; each anchor frame's file is re-hashed at capture against the manifest and the tarball inventory. | +| Manifest | The private manifest also carries a per-source digest inventory (repository-relative path and SHA-256 of every original source a final map names; no content) and the commit's Git tree object id with the dirty flag. This reconciles ADR-0111's "private per-source manifest" with packaging.md's field list; per-source inventories stay out of every public receipt. | +| Source revision | A clean candidate's anchor sources are read from the original checkout's Git objects at the recorded commit through isolated Git and must equal the manifest's digests; a dirty candidate (local only — a GitHub origin with a dirty source is already refused) uses the candidate copy's bytes, with `dirty` recorded. | +| Post-receipt rebuild | ADR-0111's "post-receipt rebuild fail" is closed in 9a.2b for the package-test producer: a second prepack into sealed staging, a re-seal into the artifact directory and a rebuilt `dist` that differs from the manifest are refused. The release-mode no-rebuild retry rules stay with 9c and 9d. | + +Engineering defaults recorded with the rulings: an in-house strict VLQ decoder in tooling, with Node's +built-in `node:module` source map used only as an independent test oracle; anchor frames must land on an +exact mapping segment, and the expected 1-based line and column come from the source revision's TypeScript +syntax tree, never from a pinned position table; the finalizer reuses the decoder for strict mapping and range +checks; the manifest is a separate owner-only `staging/manifest.json`, removed with the staging; the +`package_test` receipt keeps version 1 and its fields; consumers and the aggregate verify identity through the +shared validator and change no schema; `check:map-controls` is a standalone runner in `tools/cli-package`, +outside the causal source roots; a tag push now produces a package-test artifact that the interim guard refuses. +The inert `--verbose` raw-stack branch the preflight found is recorded for Step 10a, which replaces it. From 1ab4c1347d714644e4611ef0315ad39cb1ee540c Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sat, 24 Oct 2026 08:30:54 +0300 Subject: [PATCH 38/97] feat(repo): seal the private map manifest and symbolicate packed frames Wave 3 Step 9a.2a (#46, ADR-0111): before the package-test receipt is written, the producer now proves that the packed CLI and its inlined engine map back to their original TypeScript. - source-map.mjs decodes mappings strictly (base64 VLQ only, 1/4/5 fields, increasing columns, in-range indices) and checks every generated and original position against its text. Node's own source map decoder is lenient and serves only as a test oracle. - The finalizer decodes every upstream and final map strictly and range-checks it, under new fixed codes, before anything is written. - manifest.mjs seals the private manifest into staging after the archive inspection. It holds the commit and tree, toolchain, target, lock and config digests, dist/.js and map digests, a digest per original source and the tarball digest; no host path or content. - frame-capture.mjs is a tooling-only preload. During an offline `relavium import` of a canary workflow it wraps yaml's parse, records one synchronous stack across the CLI and @relavium/core, and refuses the keyring so no native module loads. - symbolicate.mjs maps those frames through the staged maps. Five anchors must land on exact segments at the call sites that the matching source revision's syntax tree gives: the checkout's Git objects when clean, the candidate copy when dirty, both bound to the manifest. - sealPackage runs inspection, then the manifest, then symbolication, and only then writes the receipt. The receipt stays compiled_package_test until 9a.2b's controls exist. On the real product the decoder matches Node at all 245,806 final and 330,759 upstream segment starts, and all five anchors are reached. The docs, register and status lines are updated, and the deferred 1025-character bound nit is closed. Gates: lint:tools, typecheck:tools, test:acceptance-tools (856 of 856) and smoke:cli-package with the real producer and consumer. Refs: ADR-0111 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- CLAUDE.md | 2 +- docs/reference/cli/README.md | 2 +- docs/reference/cli/packaging.md | 48 +- docs/roadmap/current.md | 4 +- docs/roadmap/deferred-tasks.md | 2 +- .../phase-2.5.5-hardening-and-remediation.md | 21 +- package.json | 2 +- tools/cli-package/README.md | 11 +- tools/cli-package/check.test.mjs | 1 + tools/cli-package/finalize.mjs | 99 ++- tools/cli-package/finalize.test.mjs | 48 ++ tools/cli-package/frame-capture-hooks.mjs | 11 + tools/cli-package/frame-capture-keyring.mjs | 6 + tools/cli-package/frame-capture.mjs | 59 ++ tools/cli-package/manifest.mjs | 296 ++++++++ tools/cli-package/pack.mjs | 3 +- tools/cli-package/prepack.test.mjs | 1 + tools/cli-package/produce.mjs | 10 +- tools/cli-package/produce.test.mjs | 20 +- tools/cli-package/seal.mjs | 28 +- tools/cli-package/seal.test.mjs | 126 +++- tools/cli-package/source-map.mjs | 181 +++++ tools/cli-package/source-map.test.mjs | 279 ++++++++ tools/cli-package/symbolicate.mjs | 346 ++++++++++ tools/cli-package/symbolicate.test.mjs | 634 ++++++++++++++++++ 26 files changed, 2199 insertions(+), 43 deletions(-) create mode 100644 tools/cli-package/frame-capture-hooks.mjs create mode 100644 tools/cli-package/frame-capture-keyring.mjs create mode 100644 tools/cli-package/frame-capture.mjs create mode 100644 tools/cli-package/manifest.mjs create mode 100644 tools/cli-package/source-map.mjs create mode 100644 tools/cli-package/source-map.test.mjs create mode 100644 tools/cli-package/symbolicate.mjs create mode 100644 tools/cli-package/symbolicate.test.mjs diff --git a/AGENTS.md b/AGENTS.md index fcf4e677..ab8d186c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -130,7 +130,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is next. `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the private manifest and symbolication) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-implementation-checkpoint--2026-10-24)). `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. See [docs/roadmap/current.md](docs/roadmap/current.md) for live status. diff --git a/CLAUDE.md b/CLAUDE.md index ec89c3af..030d76ed 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -219,7 +219,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is next. `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the private manifest and symbolication) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-implementation-checkpoint--2026-10-24)). `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. For live status, per-PR history, milestone dates, and open obligations, see the canonical diff --git a/docs/reference/cli/README.md b/docs/reference/cli/README.md index 220a1e7e..ec5c5e82 100644 --- a/docs/reference/cli/README.md +++ b/docs/reference/cli/README.md @@ -13,5 +13,5 @@ Part of [reference/](../README.md). | [commands.md](commands.md) | The full `relavium` command surface (run, chat, chat-resume/list/export, agent run, list, create, import, export, logs, gate, init, provider) plus interactive TUI vs CI JSON mode. | | [home.md](home.md) | The bare-invocation interactive **Home** — the TTY gate, the read-only management strip over `history.db`, the Home↔chat mode machine + keymap, bracketed paste, the min-terminal degrade, and the signal/exit-code lifecycle (2.5.B / ADR-0054). | | [chat-session.md](chat-session.md) | The `relavium chat` agent-session REPL — entry, agent/model selection, the multi-turn loop, streaming, tool availability, the `--json` session-event stream, and exit code 4. | -| [packaging.md](packaging.md) | Wave 3 finite runtime graph, installed-package consumers and Step 8 native activation accepted on `development`; the Step 9a.1 candidate build and finalizer are accepted on `development`; the remaining Step 9 map/custody and release gates remain staged. | +| [packaging.md](packaging.md) | Wave 3 finite runtime graph, installed-package consumers and Step 8 native activation accepted on `development`; the Step 9a.1 candidate build and finalizer are accepted on `development` and Step 9a.2a's private manifest and symbolication are implemented (acceptance pending); the remaining Step 9 map/custody and release gates remain staged. | | [agent-run-fixture.md](agent-run-fixture.md) | The `relavium agent run --fixture` cassette format — a `StreamChunk[][]` JSON recording of an agent run's LLM stream, for deterministic offline replay. | diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 6ebcfac7..1a085f2e 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -1,6 +1,6 @@ # CLI Packaging Contract -- **Status**: Compiled graph, installed package and Step 8 native activation independently accepted on `development`; the Step 9a.1 candidate build and finalizer are independently accepted on `development` (2026-10-24); the remaining Step 9 map/custody and release gates remain staged +- **Status**: Compiled graph, installed package and Step 8 native activation independently accepted on `development`; the Step 9a.1 candidate build and finalizer are independently accepted on `development` (2026-10-24); Step 9a.2a's private manifest and symbolication are implemented, with independent acceptance pending; the remaining Step 9 map/custody and release gates remain staged - **Date**: 2026-10-15 - **Related**: [ADR-0111](../../decisions/0111-cli-meta-operations-use-a-lazy-packaged-runtime.md), [commands.md](commands.md), [release runbook](../../runbooks/release-a-surface.md) @@ -331,11 +331,55 @@ section means mode 0700 for directories and 0600 for staged files, set — and f directory verified together with this process's uid — where POSIX modes are observable. On Windows no mode evidence is claimed: the candidate root and staging inherit the temporary directory's ACL, which is not verified, as the graph checks above claim no unobservable POSIX mode bits. The -private map manifest, actual packed CLI and engine symbolication, `check:map-controls` and the `package_test` receipt flip are Step 9a.2; +private map manifest and actual packed CLI and engine symbolication are Step 9a.2a (below); +`check:map-controls` and the `package_test` receipt flip are Step 9a.2b; recipient custody, the release state controller and the workflow graph are Steps 9b–9d. Until those and Wave 4b's production recipient exist, the release workflow's publish job refuses every artifact (the [Interim release guard ruling](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) of 2026-10-19). +### Private manifest and symbolication implementation boundary — Step 9a.2a + +Step 9a.2a (a [recorded split](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24) +of 9a.2, maintainer-approved on 2026-10-24) implements items 4 and 5 above for the package-test +producer. The receipt stays `compiled_package_test` until 9a.2b's product map controls exist. + +- **Strict maps.** The finalizer decodes every consumed upstream map's and every final map's + `mappings` strictly — base64 VLQ only, no truncated or oversized value, segments of one, four or + five fields, increasing generated columns, in-range source and name indices, no negative position — + and requires every generated position inside the text the map was generated for and every + original position inside its source text (a position at a line's end is a caret position). A + violation refuses with its own code (`UPSTREAM_MAP_MAPPINGS`, `UPSTREAM_MAP_RANGE`, + `MAP_MAPPINGS`, `MAP_RANGE`) before anything is written; a well-formed charset alone no longer + decides, and the bundler's silent fallback for a malformed upstream map is not relied on. Node's + built-in source map decoder is lenient, so it serves only as an independent test oracle. +- **Private manifest.** After the archive inspection the seal writes `manifest.json` into private + staging (owner-only, never overwritten): version 1; the source commit, the commit's Git tree and + the `dirty` flag; the package version; the build target; Node, pnpm, TypeScript, tsup and esbuild + versions from the installed tools; SHA-256 digests of the lockfile and every manifest and build + configuration; every final JS file keyed as `dist/.js` with its SHA-256 and its staged map's + path and SHA-256; a SHA-256 for every original source a final map names (repository-relative, no + content); and the tarball's SHA-256. The staged maps must still be the bytes the finalizer recorded + and each JS digest must be the tarball entry's. It holds no host path, credential or source text + and is removed with the staging. +- **Actual frames.** The seal then runs the finalized `dist` — which the inspection has proven + byte-equal to the tarball entry by entry — under a tooling-only `node --import` preload while it + imports a canary workflow (`relavium import`, offline, with its home, project and captured frames + in an owner-only directory of the private root). The preload wraps the external `yaml` package's + `parse`, records V8's structured call sites for the canary document alone, restores the stack + formatter and limit, and replaces the keyring binding with a refusing module, so no native module + loads and no secret is read. Exactly one captured stack is required. +- **Symbolication.** Each frame inside the finalized `dist` must be a file the manifest and the + tarball record (re-hashed at capture), its map must be the staged bytes the manifest records, and + only a frame that lands exactly on a mapping segment counts. Five named call sites must be reached + — three in the CLI and two in the inlined `@relavium/core`, each the one call to a named function + inside another — at the line and UTF-16 column that the matching source revision's syntax tree + gives: the checkout's Git objects at the recorded commit, read by Git without configuration, + filters or replace refs, for a clean candidate, and the candidate copy for a dirty local one, both + bound to the manifest's per-source digests. Wrong JS, a wrong map, permuted map sources and a wrong + source revision therefore cannot reach the anchors. A refusal prints one fixed code + (`PACKED_CLI_MANIFEST_*`, `PACKED_CLI_SYMBOLICATION_*`) and never a frame, path or source text, and + no receipt is written. + ## Recipient custody The maintainer selected encrypted CI archival. GnuPG is release tooling, never a diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index ba8c74da..aed40c00 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -172,7 +172,7 @@ bar and round cap, as its medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, by the author alone). Its other findings are recorded in -[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is independently accepted on `development` at `cf97b7f1` (2026-10-24): its round 13 reported no blocker, high or medium finding and its security sitting was clean, and the round's low findings are recorded in deferred-tasks.md ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)); Step 9a.2 is next; the substrate's +[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is independently accepted on `development` at `cf97b7f1` (2026-10-24): its round 13 reported no blocker, high or medium finding and its security sitting was clean, and the round's low findings are recorded in deferred-tasks.md ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)); Step 9a.2 is split into 9a.2a and 9a.2b, and Step 9a.2a (the private manifest and symbolication) is implemented on `development` and under review ([checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-implementation-checkpoint--2026-10-24)); the substrate's 9a.1–9d increments carry a dedicated archive/crypto security sitting. Steps 10–19 follow the [increment table](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) and end with whole-wave acceptance at Step 19. `development` is pushed after each accepted increment; PR #95 no longer bounds the @@ -1141,7 +1141,7 @@ an ESLint selector rather than a list. ### Wave 3 — Clear the ground -**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Step 9a.1 accepted on `development` (2026-10-24), Steps 9a.2–19 NOT STARTED — 2026-10-24.** +**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Step 9a.1 accepted on `development` (2026-10-24), Step 9a.2a under review, Steps 9a.2b–19 NOT STARTED — 2026-10-24.** ADR-0110–0113, their linked contracts and the nineteen-increment plan were approved on 2026-10-16. [Steps 1–8 were independently accepted](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) on `development`, with whole Step 8 reviewed at `c83c8575`, and published as the scoped checkpoint diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index 0f8c0d5c..4bf11076 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -627,7 +627,7 @@ documentation-only ones were also corrected in the acceptance commit, which chan detached into its own group, so the real composition behaves the same. **Re-check:** in the forwarding harness, have the detached child start a non-detached grandchild in its group that writes a marker on SIGINT, and assert both markers. *(low · tools/cli-package/process.mjs)* -- [ ] **The map-source length bound is pinned on its admitted side only.** *Nit.* *(round 13, TCA-9A1-R13-2)* +- [x] **The map-source length bound is pinned on its admitted side only.** *Nit; closed in Step 9a.2a (2026-10-24), whose finalizer tests add the 1025-character source.* *(round 13, TCA-9A1-R13-2)* `finalize.test.mjs` admits a 1024-character source and refuses a 1031-character one, so loosening `source.length > 1024` to any bound up to 1030 passes. The source must still resolve to an existing original file, so nothing is exposed. **Re-check:** add a 1025-character source (`../src/${'a/'.repeat(507)}x.ts`) diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 8327a873..5ba2a9b8 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -3699,7 +3699,7 @@ No credential values or request captures belong in tracked evidence. | 9a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage #46/ADR-0111 unchanged, no new contract) — release maps, final build and symbolication:** fresh uncached upstream original-TypeScript maps, the sole prepack final CLI build, final trailer removal before hashes, map and private-graph staging outside public artifacts and caches, exact tarball-to-final-JS binding, and real packed CLI and engine frame symbolication with wrong JS/map/source and missing-map negative controls. Ordinary builds remain map-free. *Split again on 2026-10-19 into 9a.1 and 9a.2 below, which replace it as increments ([Plan shape](#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19)).* | | 9a.1 | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — candidate build and finalizer:** an owned isolated candidate copy, fresh uncached upstream maps, candidate-mode tsup selection with absent `sourcesContent`, the prepack finalizer (upstream and final map validation, repository-relative sources, exact trailer removal before hashes), owner-only private staging outside public artifacts and caches, closure/seal adaptation, the ordinary map-free control, the PR #95 archive/worker fold-ins, and a fail-closed `release.yml` publish guard that refuses any non-release candidate until 9d and Wave 4b custody exist. The receipt stays `compiled_package_test`. *Implemented on `development` on 2026-10-22 ([checkpoint](#wave-3-step-9a1-implementation-checkpoint--2026-10-22)); independently accepted at `cf97b7f1` on 2026-10-24 ([acceptance](#wave-3-step-9a1-acceptance--2026-10-24)).* | | 9a.2 | **Recorded split — manifest, symbolication and package-test receipt:** the private map manifest keyed by `dist/.js`, actual packed CLI and engine frame symbolication against the matching source revision, wrong JS/map/source and missing/malformed-upstream negative controls (product copies via `check:map-controls`, also a required uncached CI job), and the atomic `package_test` receipt flip across seal, binding, consumers, aggregate and both workflows. *Split on 2026-10-24 (maintainer-approved; ID coverage #46/ADR-0111 unchanged) into 9a.2a and 9a.2b below, which replace it as increments ([preflight decisions](#wave-3-step-9a2-preflight-decisions--2026-10-24)).* | -| 9a.2a | **Recorded split, 2026-10-24 (maintainer-approved) — decoder, private manifest and symbolication:** an in-house strict source-map decoder; strict mapping and range validation of upstream and final maps in the finalizer; the private map manifest (`staging/manifest.json`, keyed by `dist/.js`, with a per-source digest inventory and the commit's tree), sealed after the archive inspection; real packed CLI and `@relavium/core` frame capture through a tooling-only preload during an offline `relavium import`, run on the finalized `dist` that the inspection proves byte-equal to the tarball; symbolication of those frames against the matching source revision (clean: the checkout's Git objects; dirty, local only: the candidate copy), wired into the seal before the receipt is written. The receipt stays `compiled_package_test`. Security sitting. | +| 9a.2a | **Recorded split, 2026-10-24 (maintainer-approved) — decoder, private manifest and symbolication:** an in-house strict source-map decoder; strict mapping and range validation of upstream and final maps in the finalizer; the private map manifest (`staging/manifest.json`, keyed by `dist/.js`, with a per-source digest inventory and the commit's tree), sealed after the archive inspection; real packed CLI and `@relavium/core` frame capture through a tooling-only preload during an offline `relavium import`, run on the finalized `dist` that the inspection proves byte-equal to the tarball; symbolication of those frames against the matching source revision (clean: the checkout's Git objects; dirty, local only: the candidate copy), wired into the seal before the receipt is written. The receipt stays `compiled_package_test`. Security sitting. *Implemented on `development` on 2026-10-24; acceptance pending ([checkpoint](#wave-3-step-9a2a-implementation-checkpoint--2026-10-24)).* | | 9a.2b | **Recorded split, 2026-10-24 (maintainer-approved) — product map controls and the package-test receipt:** `check:map-controls` over product copies (wrong JS by line and column, wrong map, permuted sources, wrong source, missing, malformed and truncated upstream maps, a shifted valid upstream map, and the package-test producer's post-receipt rebuild refusals that ADR-0111 lists), run in `pnpm run ci` and as a required uncached CI job inside `ci`'s needs (the required contexts stay `ci` and `coverage`); then the atomic `package_test` flip across seal, binding, consumers, aggregate and both workflows, version 1 kept. Security sitting. | | 9b | **Recorded split — recipient custody:** synthetic-only isolated GnuPG controls (public-only home/keyring, pinned primary and encryption-subkey fingerprints, machine-readable refusal of missing, expired, revoked or unusable recipients), encrypt once and reuse the original ciphertext. Explicit crypto/custody security sitting. Fixture proof never claims production recipient or custody configuration. | | 9c | **Recorded split — offline release state controller:** permission-enforcing offline GitHub/npm controller, permissioned prebuild discovery and reservation, and every rerun, partial, conflict and expiry state, with no overwrite, rebuild or re-encryption on uncertain history. | @@ -9682,3 +9682,22 @@ checks; the manifest is a separate owner-only `staging/manifest.json`, removed w shared validator and change no schema; `check:map-controls` is a standalone runner in `tools/cli-package`, outside the causal source roots; a tag push now produces a package-test artifact that the interim guard refuses. The inert `--verbose` raw-stack branch the preflight found is recorded for Step 10a, which replaces it. + +### Wave 3 Step 9a.2a implementation checkpoint — 2026-10-24 + +**Scope and source.** Step 9a.2a (the 9a.2a row; #46, ADR-0111) implements, on `development` after the 9a.1 +acceptance and the 9a.2 preflight record (`6277e375`), the strict source-map decoder (`tools/cli-package/source-map.mjs`), +the finalizer's strict mapping and range checks (`finalize.mjs`), the private manifest (`manifest.mjs`), the +tooling-only frame-capture preload with its keyring stub (`frame-capture.mjs`, `frame-capture-hooks.mjs`, +`frame-capture-keyring.mjs`), symbolication against the matching source revision (`symbolicate.mjs`), the seal's +composition (inspection → manifest → symbolication → receipt, `seal.mjs`), the producer's private work directory +(`produce.mjs`) and the entry point's summary (`pack.mjs`), with their tests and the +[packaging contract](../../reference/cli/packaging.md#private-manifest-and-symbolication-implementation-boundary--step-9a2a), +the tooling README and the status lines. No runtime dependency is added, Node's own source map decoder is only a +test oracle, and the receipt stays `compiled_package_test`. On the real product the decoder agrees with that +oracle at all 245,806 final-map and 330,759 upstream-map segment starts, every position lies inside its text, and +the real producer seals a 25-output, 404-source manifest and reaches all five anchors in about a second. + +**Remaining gates.** The internal rigor pass (whole-increment mutation sweep, claims, platform and security +lenses), the full `pnpm run ci`, then one NEW complete cumulative round with no blocker, high or medium finding +and no regression, together with the separate scoped security sitting at the same head and bar. \ No newline at end of file diff --git a/package.json b/package.json index d7604427..8bb79e76 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "smoke:budget-replay": "node tools/budget-replay-compat/check.mjs", "check:wire-mutations": "node tools/wire-conformance/check.mjs", "check:usage-mutations": "node tools/usage-coverage-controls/check.mjs", - "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/check.test.mjs tools/cli-package/paths.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", + "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/check.test.mjs tools/cli-package/paths.test.mjs tools/cli-package/source-map.test.mjs tools/cli-package/symbolicate.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", "check:fs-mutations": "node tools/fs-policy-controls/check.mjs", "check:secrets-mutations": "node tools/secrets-controls/check.mjs", "smoke:cli-bootstrap": "node tools/cli-bootstrap/check.mjs", diff --git a/tools/cli-package/README.md b/tools/cli-package/README.md index 760a78c1..d88fdcd3 100644 --- a/tools/cli-package/README.md +++ b/tools/cli-package/README.md @@ -58,9 +58,14 @@ consumer installs under the temporary directory, so a shared `/tmp` is a named l production prints only prepack's own fixed `✗ CLI …` lines and one fixed code (`production-failure.mjs`), never a build child's captured output, which can quote a map. A corepack-managed pnpm always runs from the developer's own corepack cache, never from the shared -temporary directory that holds the isolated homes (`corepackHome` in `process.mjs`). The receipt does **not** certify symbolication, a map manifest, -encrypted custody or a production release candidate (Steps 9a.2–9d). No private map, -metafile or per-source inventory enters the public artifact. The release workflow's +temporary directory that holds the isolated homes (`corepackHome` in `process.mjs`). Before the +receipt is written, the seal decodes the private maps strictly (`source-map.mjs`; the finalizer does +too), seals the private manifest (`manifest.mjs`) and symbolicates actual packed CLI and core +frames, captured by a tooling-only preload during an offline import (`frame-capture.mjs`), against +the matching source revision (`symbolicate.mjs`); all of it is discarded with the staging. The +receipt does **not** certify encrypted custody or a production release candidate (Steps 9b–9d), +and it stays `compiled_package_test` until the product map controls of Step 9a.2b. No private map, +metafile, manifest or per-source inventory enters the public artifact. The release workflow's `release-guard.mjs` refuses every artifact until verified release custody exists: it sets its exit code to 1 before it evaluates its rule (`release-guard-core.mjs`), so a rule that returns, throws or cannot load still refuses, and no way of invoking it can skip the diff --git a/tools/cli-package/check.test.mjs b/tools/cli-package/check.test.mjs index 55e23dbc..735a347a 100644 --- a/tools/cli-package/check.test.mjs +++ b/tools/cli-package/check.test.mjs @@ -35,6 +35,7 @@ const COPIED = Object.freeze([ 'cli-package/consumer-lifetime.mjs', 'cli-package/files.mjs', 'cli-package/finalize.mjs', + 'cli-package/source-map.mjs', 'cli-package/paths.mjs', 'cli-package/process.mjs', 'bundle-closure/closure.mjs', diff --git a/tools/cli-package/finalize.mjs b/tools/cli-package/finalize.mjs index 291b8622..884cb728 100644 --- a/tools/cli-package/finalize.mjs +++ b/tools/cli-package/finalize.mjs @@ -1,10 +1,12 @@ /** - * Candidate finalizer (ADR-0111, Step 9a.1). It runs inside the CLI's prepack after the sole - * candidate tsup build and before pnpm packs. It accounts for every emitted output, validates every - * consumed upstream TypeScript map and every final map, rewrites final map sources to - * repository-relative original TypeScript, removes only tsup's exact terminal map-link trailers, - * stages the maps and the build metadata owner-only outside the package, re-checks the finite - * closure and the public file/mode inventory, and only then hashes. Nothing writes JS afterwards. + * Candidate finalizer (ADR-0111, Steps 9a.1 and 9a.2a). It runs inside the CLI's prepack after the + * sole candidate tsup build and before pnpm packs. It accounts for every emitted output, validates + * every consumed upstream TypeScript map and every final map — decoding their mappings strictly and + * keeping every position inside its generated and original text (source-map.mjs) — rewrites final + * map sources to repository-relative original TypeScript, removes only tsup's exact terminal map-link + * trailers, stages the maps and the build metadata owner-only outside the package, re-checks the + * finite closure and the public file/mode inventory, and only then hashes. Nothing writes JS + * afterwards. * * Refusals name a fixed code and at most a fixed label or a repository-relative path that has passed * validation, never map or source contents and never a host path: a metafile string that escapes @@ -33,6 +35,13 @@ import { checkBundleClosure } from '../bundle-closure/closure.mjs'; import { MODE_EVIDENCE, emittedInventory } from '../bundle-closure/inventory.mjs'; import { flatFiles, readOrdinaryFile, sha256 } from './files.mjs'; import { within } from './paths.mjs'; +import { + SourceMapError, + assertGeneratedRange, + assertOriginalRange, + decodeMappings, + lineLengths, +} from './source-map.mjs'; /** * Names the private staging directory of a trusted candidate build. Only the package producer sets @@ -160,6 +169,46 @@ function originalSource(repository, mapDirectory, source, allowedRoots, subject) return resolved; } +/** + * Decode a map's mappings strictly and require every generated position inside the text it was + * generated for and every original position inside its source: a well-formed charset alone admits + * truncated values, out-of-range indices and positions past a file's end, which the bundler would + * otherwise drop or chain silently. + * @param {string} repository + * @param {{mappings: string, sources: unknown[], names: unknown[]}} map + * @param {string} generated the generated text + * @param {readonly string[]} sources repository-relative original sources, by index + * @param {Map} texts source line lengths, shared by every map of one run + * @param {'UPSTREAM_MAP' | 'MAP'} kind @param {string} subject + */ +function checkMappings(repository, map, generated, sources, texts, kind, subject) { + /** @param {() => void} check @param {string} code */ + const strictly = (check, code) => { + try { + check(); + } catch (error) { + if (error instanceof SourceMapError) refuse(code, subject); + throw error; + } + }; + let lines = []; + strictly(() => { + lines = decodeMappings(map.mappings, { sources: map.sources.length, names: map.names.length }); + }, `${kind}_MAPPINGS`); + const originals = sources.map((source) => { + let lengths = texts.get(source); + if (lengths === undefined) { + lengths = lineLengths(readText(onDisk(repository, source), source)); + texts.set(source, lengths); + } + return lengths; + }); + strictly(() => { + assertGeneratedRange(lines, lineLengths(generated)); + assertOriginalRange(lines, originals); + }, `${kind}_RANGE`); +} + /** * Validate one consumed upstream (tsc) map. The JS must end with its exact trailer and the map must * name only this package's original TypeScript, without embedded sources or a source root. Its @@ -168,8 +217,9 @@ function originalSource(repository, mapDirectory, source, allowedRoots, subject) * @param {string} repository * @param {string} packageName * @param {string} jsPath repository-relative POSIX path of the consumed compiled JS + * @param {Map} texts source line lengths, shared by every map of one run */ -function upstreamSources(repository, packageName, jsPath) { +function upstreamSources(repository, packageName, jsPath, texts) { const name = posix.basename(jsPath); const js = readText(onDisk(repository, jsPath), jsPath); if (!js.endsWith(mapTrailer(name)) || js.split('sourceMappingURL').length !== 2) { @@ -211,9 +261,11 @@ function upstreamSources(repository, packageName, jsPath) { refuse('UPSTREAM_MAP_UNMAPPED', mapPath); } const directory = posix.dirname(jsPath); - return map.sources.map((source) => + const sources = map.sources.map((source) => originalSource(repository, directory, source, [`packages/${packageName}/src`], mapPath), ); + checkMappings(repository, map, js, sources, texts, 'UPSTREAM_MAP', mapPath); + return sources; } /** @@ -232,8 +284,9 @@ function inputSubject(input) { * package's compiled JS through its tsc map. Any other input kind refuses. * @param {string} repository @param {string} input metafile input relative to apps/cli * @param {Map} cache + * @param {Map} texts source line lengths, shared by every map of one run */ -function inputSources(repository, input, cache) { +function inputSources(repository, input, cache, texts) { const cached = cache.get(input); if (cached) return cached; let sources; @@ -247,7 +300,12 @@ function inputSources(repository, input, cache) { ENGINE_PACKAGES.includes(engine[1]) && /^(?:[A-Za-z0-9_-][A-Za-z0-9_.-]*\/)*[A-Za-z0-9_-][A-Za-z0-9_.-]*\.js$/u.test(engine[2]) ) { - sources = upstreamSources(repository, engine[1], `packages/${engine[1]}/dist/${engine[2]}`); + sources = upstreamSources( + repository, + engine[1], + `packages/${engine[1]}/dist/${engine[2]}`, + texts, + ); } else { refuse('UNSUPPORTED_INPUT', inputSubject(input)); } @@ -259,8 +317,10 @@ function inputSources(repository, input, cache) { * Validate one final map against the original sources of the inputs that contributed bytes to its * runtime output, and return it with repository-relative sources. Mappings and names are kept. * @param {string} repository @param {string} jsName @param {unknown} map @param {Set} expected + * @param {string} code the runtime output without its map-link trailer + * @param {Map} texts source line lengths, shared by every map of one run */ -function finalMap(repository, jsName, map, expected) { +function finalMap(repository, jsName, map, expected, code, texts) { const subject = `apps/cli/dist/${jsName}.map`; const keys = isRecord(map) ? Object.keys(map) : []; if ( @@ -305,6 +365,7 @@ function finalMap(repository, jsName, map, expected) { if (!isDeepStrictEqual([...sources].sort(), [...expected].sort())) { refuse('MAP_SOURCE_COVERAGE', subject); } + checkMappings(repository, map, code, sources, texts, 'MAP', subject); return { ...map, sources }; } @@ -360,8 +421,8 @@ function writeNew(path, bytes, mode) { } } -/** Write bytes to a new owner-only staging file. */ -function writePrivate(path, bytes) { +/** Write bytes to a new owner-only staging file; an existing file refuses. */ +export function writePrivateFile(path, bytes) { writeNew(path, bytes, 0o600); } @@ -413,7 +474,8 @@ export function finalizeCandidate({ repository, staging }) { // 2. Validate every consumed upstream map and every final map, and strip each exact trailer. // No file is written until every output has been accepted. const cache = new Map(); - for (const input of Object.keys(metafile.inputs)) inputSources(repository, input, cache); + const texts = new Map(); + for (const input of Object.keys(metafile.inputs)) inputSources(repository, input, cache, texts); const prepared = runtime.map((path) => { const jsName = basename(path); const output = metafile.outputs[path]; @@ -443,6 +505,8 @@ export function finalizeCandidate({ repository, staging }) { jsName, readJson(join(dist, `${jsName}.map`), LIMITS.finalMapBytes, `apps/cli/${path}.map`), expected, + js.slice(0, -trailer.length), + texts, ); const mode = lstatSync(join(dist, jsName)).mode & 0o777; return { path, jsName, code, map: Buffer.from(JSON.stringify(map), 'utf8'), mode }; @@ -452,8 +516,9 @@ export function finalizeCandidate({ repository, staging }) { mkdirSync(join(staging, 'maps'), { mode: 0o700 }); // The creation mode passes through the umask; the staged directory keeps the exact owner-only mode. if (MODE_EVIDENCE === 'posix') chmodSync(join(staging, 'maps'), 0o700); - for (const item of prepared) writePrivate(join(staging, 'maps', `${item.jsName}.map`), item.map); - writePrivate(join(staging, 'metafile-esm.json'), metafileBytes); + for (const item of prepared) + writePrivateFile(join(staging, 'maps', `${item.jsName}.map`), item.map); + writePrivateFile(join(staging, 'metafile-esm.json'), metafileBytes); for (const item of prepared) replaceOutput(join(dist, item.jsName), item.code, item.mode); for (const item of prepared) unlinkSync(join(dist, `${item.jsName}.map`)); unlinkSync(join(dist, 'metafile-esm.json')); @@ -487,7 +552,7 @@ export function finalizeCandidate({ repository, staging }) { })); const upstreamMaps = [...cache.keys()].filter((input) => !input.startsWith('src/')).length; const record = { version: 1, outputs, maps, upstreamMaps }; - writePrivate( + writePrivateFile( join(staging, 'finalized.json'), Buffer.from(JSON.stringify(record, null, 2) + '\n'), ); diff --git a/tools/cli-package/finalize.test.mjs b/tools/cli-package/finalize.test.mjs index d375ee13..c8b58ccf 100644 --- a/tools/cli-package/finalize.test.mjs +++ b/tools/cli-package/finalize.test.mjs @@ -710,6 +710,12 @@ test('a map source must be a plain path of at most 1024 characters to an origina assert.equal(longest.length, 1024); editJson(join(bounded.core, 'dist', 'index.js.map'), (map) => (map.sources = [longest])); refusesBeforeWriting(bounded, 'MAP_SOURCE_MISSING'); + // One character more is refused as a path, before any lookup. + const over = await fixture(); + const tooLong = `../src/${'a/'.repeat(507)}x.ts`; + assert.equal(tooLong.length, 1025); + editJson(join(over.core, 'dist', 'index.js.map'), (map) => (map.sources = [tooLong])); + refusesBeforeWriting(over, 'MAP_SOURCE_PATH'); // A source under a dot directory is not original TypeScript, even when the file exists. const dotted = await fixture(); write(join(dotted.core, 'src', '.hidden', 'x.ts'), 'export const x = 1;\n'); @@ -731,6 +737,48 @@ test('a map source must be a plain path of at most 1024 characters to an origina } }); +/** Base64 VLQ for one signed value, as a source map writer encodes it. */ +function vlq(value) { + const digits = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; + let rest = value < 0 ? -value * 2 + 1 : value * 2; + let out = ''; + do { + let digit = rest % 32; + rest = Math.floor(rest / 32); + if (rest > 0) digit += 32; + out += digits[digit]; + } while (rest > 0); + return out; +} + +test('refuses mappings that are malformed or point outside their texts, upstream and final', async () => { + // Each mapping string passes the character check; only strict decoding or a range check refuses it. + const pastLastLine = `AAAA;${';'.repeat(5000)}AAAA`; + for (const [label, mappings, code] of [ + ['a truncated value', 'AAAg', 'MAPPINGS'], + ['a two-field segment', 'AA', 'MAPPINGS'], + ['a source index past the sources', `A${vlq(1)}AA`, 'MAPPINGS'], + ['an empty segment', 'AAAA,,CAAC', 'MAPPINGS'], + ['a generated column that goes back', 'EAAA,DAAC', 'MAPPINGS'], + ['an original line past the source', `AA${vlq(100000)}A`, 'RANGE'], + ['an original column past its line', `AAA${vlq(100000)}`, 'RANGE'], + ['a generated line past the text', pastLastLine, 'RANGE'], + ['a generated column past its line', `${vlq(100000)}AAA`, 'RANGE'], + ]) { + const upstream = await fixture(); + editJson(join(upstream.core, 'dist', 'index.js.map'), (map) => (map.mappings = mappings)); + const upstreamError = refusal(upstream); + assert.equal(upstreamError.code, `UPSTREAM_MAP_${code}`, `upstream ${label}`); + assert.equal( + upstreamError.message, + `CLI candidate finalization refused: UPSTREAM_MAP_${code} (packages/core/dist/index.js.map)`, + ); + const final = await fixture(); + editJson(join(final.dist, 'index.js.map'), (map) => (map.mappings = mappings)); + refusesBeforeWriting(final, `MAP_${code}`); + } +}); + test('admits original .tsx CLI sources', async () => { const candidate = await fixture({ tsx: true }); finalizeCandidate({ repository: candidate.root, staging: candidate.staging }); diff --git a/tools/cli-package/frame-capture-hooks.mjs b/tools/cli-package/frame-capture-hooks.mjs new file mode 100644 index 00000000..dfe1e796 --- /dev/null +++ b/tools/cli-package/frame-capture-hooks.mjs @@ -0,0 +1,11 @@ +// Loader hook of the symbolication preload (frame-capture.mjs): the packed CLI's keyring binding is +// replaced by a module that refuses every use, so the run loads no native module and reads no secret. +export async function resolve(specifier, context, nextResolve) { + if (specifier === '@napi-rs/keyring') + return { + url: new URL('./frame-capture-keyring.mjs', import.meta.url).href, + shortCircuit: true, + }; + if (specifier.startsWith('@napi-rs/keyring-')) throw new Error('Native keyring import refused'); + return nextResolve(specifier, context); +} diff --git a/tools/cli-package/frame-capture-keyring.mjs b/tools/cli-package/frame-capture-keyring.mjs new file mode 100644 index 00000000..21daabda --- /dev/null +++ b/tools/cli-package/frame-capture-keyring.mjs @@ -0,0 +1,6 @@ +// The keyring binding seen by the symbolication run (frame-capture-hooks.mjs): every use refuses. +export class Entry { + constructor() { + throw new Error('The symbolication run refuses the keyring'); + } +} diff --git a/tools/cli-package/frame-capture.mjs b/tools/cli-package/frame-capture.mjs new file mode 100644 index 00000000..58d92a84 --- /dev/null +++ b/tools/cli-package/frame-capture.mjs @@ -0,0 +1,59 @@ +/** + * Tooling-only preload for symbolication (ADR-0111, Step 9a.2a; the 2026-10-24 frame ruling). It runs + * with `node --import` around the packed CLI while it imports a canary workflow, and adds no product + * surface: the packed bytes are untouched. It wraps the external `yaml` package's `parse` — the one + * instance the CLI's import and require both reach — and records V8's structured call sites for the + * canary document alone, so one synchronous stack crosses the CLI and the inlined `@relavium/core`. It + * writes one owner-only record when the process exits; the producer refuses unless exactly one stack + * was captured. The keyring binding is replaced by a refusing module (frame-capture-hooks.mjs). + */ +import { closeSync, openSync, writeSync } from 'node:fs'; +import { createRequire, register } from 'node:module'; + +const log = process.env.RELAVIUM_FRAME_CAPTURE_LOG; +const manifest = process.env.RELAVIUM_FRAME_CAPTURE_PACKAGE; +const canary = process.env.RELAVIUM_FRAME_CAPTURE_CANARY; +if (!log || !manifest || !canary) throw new Error('The symbolication preload is not configured'); + +register(new URL('./frame-capture-hooks.mjs', import.meta.url)); + +/** @type {{file: string | null, line: number | null, column: number | null, async: boolean}[][]} */ +const captures = []; + +/** Structured call sites above `skip`, with the stack trace limit and formatter restored after. */ +function callSites(skip) { + const prepare = Error.prepareStackTrace; + const limit = Error.stackTraceLimit; + try { + Error.stackTraceLimit = 64; + Error.prepareStackTrace = (_error, sites) => sites; + const holder = {}; + Error.captureStackTrace(holder, skip); + return holder.stack.map((site) => ({ + file: site.getFileName() ?? null, + line: site.getLineNumber(), + column: site.getColumnNumber(), + async: site.isAsync(), + })); + } finally { + Error.prepareStackTrace = prepare; + Error.stackTraceLimit = limit; + } +} + +const yaml = createRequire(manifest)('yaml'); +const parse = yaml.parse; +if (typeof parse !== 'function') throw new Error('The yaml package has no parse export'); +yaml.parse = function capturedParse(text, ...rest) { + if (typeof text === 'string' && text.includes(canary)) captures.push(callSites(capturedParse)); + return Reflect.apply(parse, this, [text, ...rest]); +}; + +process.on('exit', () => { + const descriptor = openSync(log, 'wx', 0o600); + try { + writeSync(descriptor, JSON.stringify({ version: 1, captures })); + } finally { + closeSync(descriptor); + } +}); diff --git a/tools/cli-package/manifest.mjs b/tools/cli-package/manifest.mjs new file mode 100644 index 00000000..88e85f50 --- /dev/null +++ b/tools/cli-package/manifest.mjs @@ -0,0 +1,296 @@ +/** + * The private map manifest (ADR-0111; packaging.md, "private map manifest"; Step 9a.2a). Sealed into + * private staging only after the archive inspection, it binds the source commit and its tree, the + * package version, the toolchain and target, the lock and build-configuration digests, every final JS + * file as `dist/.js` with its map, a digest of every original source a final map names, and the + * tarball digest. It holds no host path, credential or source content, and it is removed with the + * staging; Step 9b carries it into ciphertext. + */ +import { Buffer } from 'node:buffer'; +import { createRequire } from 'node:module'; +import { basename, join } from 'node:path'; +import { isDeepStrictEqual } from 'node:util'; +import { ENGINE_PACKAGES, writePrivateFile } from './finalize.mjs'; +import { readOrdinaryFile, sha256 } from './files.mjs'; +import { isolatedGitEnvironment, runCommand } from './process.mjs'; + +export const MANIFEST_FILE = 'manifest.json'; + +/** The candidate files whose bytes decide the build: the lock and every manifest and build configuration. */ +export const MANIFEST_INPUTS = Object.freeze([ + '.npmrc', + 'package.json', + 'pnpm-lock.yaml', + 'pnpm-workspace.yaml', + 'tsconfig.base.json', + 'apps/cli/package.json', + 'apps/cli/tsconfig.json', + 'apps/cli/tsup.config.ts', + ...ENGINE_PACKAGES.flatMap((name) => [ + `packages/${name}/package.json`, + `packages/${name}/tsconfig.build.json`, + ]), +]); + +const SHA256 = /^[0-9a-f]{64}$/u; +const GIT_OBJECT = /^[0-9a-f]{40}$/u; +const OUTPUT = /^dist\/[A-Za-z0-9_-]+\.js$/u; +const SEGMENT = '[A-Za-z0-9_-][A-Za-z0-9_.-]*'; +/** A repository-relative original source the finalizer admits: CLI or engine TypeScript. */ +const SOURCE = new RegExp( + `^(?:apps/cli|packages/(?:${ENGINE_PACKAGES.join('|')}))/src/(?:${SEGMENT}/)*${SEGMENT}\\.tsx?$`, + 'u', +); +const VERSION = /^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]{1,64})?$/u; +const LIMITS = Object.freeze({ outputs: 256, sources: 8192, recordBytes: 1024 * 1024 }); + +export class ManifestError extends Error { + /** @param {string} code */ + constructor(code) { + super(`PACKED_CLI_MANIFEST_${code}`); + this.name = 'ManifestError'; + this.code = code; + } +} + +/** @param {string} code @returns {never} */ +function refuse(code) { + throw new ManifestError(code); +} + +/** @param {string} root @param {string} path repository-relative POSIX path @param {number} limit */ +function read(root, path, limit) { + try { + return readOrdinaryFile(join(root, ...path.split('/')), limit); + } catch { + refuse('UNREADABLE_INPUT'); + } +} + +/** @param {unknown} value @returns {value is Record} */ +function isRecord(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +/** + * @typedef {{version: 1, source: {commit: string, tree: string, dirty: boolean}, + * packageVersion: string, target: string, + * toolchain: {node: string, pnpm: string, typescript: string, tsup: string, esbuild: string}, + * inputs: Record, + * outputs: {path: string, sha256: string, map: {path: string, sha256: string}}[], + * sources: Record, tarballSha256: string}} PrivateManifest + */ + +/** + * Validate a manifest's exact shape. Every key is fixed, every digest is SHA-256 hex, every path is + * repository- or package-relative, and the outputs and sources are sorted and unique. + * @param {unknown} value + * @returns {PrivateManifest} + */ +export function validateManifest(value) { + const exact = (record, keys) => + isRecord(record) && isDeepStrictEqual(Object.keys(record).sort(), [...keys].sort()); + if ( + !exact(value, [ + 'version', + 'source', + 'packageVersion', + 'target', + 'toolchain', + 'inputs', + 'outputs', + 'sources', + 'tarballSha256', + ]) || + value.version !== 1 + ) { + refuse('SHAPE'); + } + const { source, toolchain, inputs, outputs, sources } = value; + if ( + !exact(source, ['commit', 'tree', 'dirty']) || + !GIT_OBJECT.test(String(source.commit)) || + !GIT_OBJECT.test(String(source.tree)) || + typeof source.dirty !== 'boolean' || + typeof value.packageVersion !== 'string' || + !VERSION.test(value.packageVersion) || + typeof value.target !== 'string' || + !/^node\d{2,3}$/u.test(value.target) || + !exact(toolchain, ['node', 'pnpm', 'typescript', 'tsup', 'esbuild']) || + !/^v\d+\.\d+\.\d+$/u.test(String(toolchain.node)) || + toolchain.pnpm !== '9.12.3' || + !['typescript', 'tsup', 'esbuild'].every((name) => VERSION.test(String(toolchain[name]))) || + !exact(inputs, MANIFEST_INPUTS) || + !Object.values(inputs).every((digest) => SHA256.test(String(digest))) || + typeof value.tarballSha256 !== 'string' || + !SHA256.test(value.tarballSha256) + ) { + refuse('SHAPE'); + } + if ( + !Array.isArray(outputs) || + outputs.length === 0 || + outputs.length > LIMITS.outputs || + !outputs.every( + (output, index) => + exact(output, ['path', 'sha256', 'map']) && + OUTPUT.test(String(output.path)) && + SHA256.test(String(output.sha256)) && + exact(output.map, ['path', 'sha256']) && + output.map.path === `${output.path}.map` && + SHA256.test(String(output.map.sha256)) && + (index === 0 || outputs[index - 1].path < output.path), + ) + ) { + refuse('SHAPE'); + } + const names = isRecord(sources) ? Object.keys(sources) : []; + if ( + !isRecord(sources) || + names.length === 0 || + names.length > LIMITS.sources || + !names.every( + (name, index) => + SOURCE.test(name) && + SHA256.test(String(sources[name])) && + (index === 0 || names[index - 1] < name), + ) + ) { + refuse('SHAPE'); + } + return /** @type {PrivateManifest} */ (value); +} + +/** + * Seal the private manifest into `staging`, after the archive inspection has proven the tarball's + * JS byte-equal to the finalized files. The staged maps must still be the bytes the finalizer + * recorded, and each recorded JS digest must be the tarball entry's. An existing manifest refuses. + * @param {{candidate: string, staging: string, checkout: string, source: string, dirty: boolean, + * archive: {sha256: string, files: {path: string, sha256: string}[], package: {version: string}}, + * run?: typeof runCommand}} options + * @returns {Promise} + */ +export async function sealPrivateManifest({ + candidate, + staging, + checkout, + source, + dirty, + archive, + run = runCommand, +}) { + let record; + try { + record = JSON.parse( + readOrdinaryFile(join(staging, 'finalized.json'), LIMITS.recordBytes).toString('utf8'), + ); + } catch { + refuse('RECORD'); + } + if (!isRecord(record) || !Array.isArray(record.outputs) || !Array.isArray(record.maps)) { + refuse('RECORD'); + } + const maps = new Map(record.maps.map((map) => [isRecord(map) ? map.path : undefined, map])); + const packed = new Map(archive.files.map((file) => [file.path, file.sha256])); + const named = new Set(); + const outputs = record.outputs.map((output) => { + const map = isRecord(output) ? maps.get(`${output.path}.map`) : undefined; + if (!isRecord(output) || !isRecord(map) || !OUTPUT.test(String(output.path))) refuse('RECORD'); + const bytes = read(staging, `maps/${basename(output.path)}.map`, 16 * 1024 * 1024); + if (bytes.length !== map.bytes || sha256(bytes) !== map.sha256) refuse('MAP_DRIFT'); + if (packed.get(`package/${output.path}`) !== output.sha256) refuse('TARBALL_DRIFT'); + let parsed; + try { + parsed = JSON.parse(bytes.toString('utf8')); + } catch { + refuse('MAP_DRIFT'); + } + if (!isRecord(parsed) || !Array.isArray(parsed.sources)) refuse('MAP_DRIFT'); + for (const name of parsed.sources) named.add(name); + return { + path: output.path, + sha256: output.sha256, + map: { path: map.path, sha256: map.sha256 }, + }; + }); + if (maps.size !== outputs.length) refuse('RECORD'); + outputs.sort((left, right) => (left.path < right.path ? -1 : 1)); + const sources = Object.fromEntries( + [...named] + .map(String) + .sort() + .map((name) => { + if (!SOURCE.test(name)) refuse('SOURCE_PATH'); + return [name, sha256(read(candidate, name, 8 * 1024 * 1024))]; + }), + ); + let tree; + try { + tree = ( + await run('git', ['--no-replace-objects', 'rev-parse', '--verify', `${source}^{tree}`], { + cwd: checkout, + env: isolatedGitEnvironment(), + timeoutMs: 60_000, + }) + ).stdout.trim(); + } catch { + refuse('SOURCE_TREE'); + } + if (!GIT_OBJECT.test(tree)) refuse('SOURCE_TREE'); + const cli = createRequire(join(candidate, 'apps', 'cli', 'package.json')); + const version = (require, name) => { + try { + return String(require(`${name}/package.json`).version); + } catch { + refuse('TOOLCHAIN'); + } + }; + const toolchain = { + node: process.version, + pnpm: '9.12.3', + typescript: version(cli, 'typescript'), + tsup: version(cli, 'tsup'), + esbuild: version(createRequire(cli.resolve('tsup/package.json')), 'esbuild'), + }; + const targets = [ + ...read(candidate, 'apps/cli/tsup.config.ts', 128 * 1024) + .toString('utf8') + .matchAll(/^ {2}target: '([^']+)',$/gmu), + ]; + if (targets.length !== 1) refuse('TARGET'); + const manifest = validateManifest({ + version: 1, + source: { commit: source, tree, dirty }, + packageVersion: archive.package.version, + target: targets[0][1], + toolchain, + inputs: Object.fromEntries( + MANIFEST_INPUTS.map((path) => [path, sha256(read(candidate, path, 16 * 1024 * 1024))]), + ), + outputs, + sources, + tarballSha256: archive.sha256, + }); + writePrivateFile( + join(staging, MANIFEST_FILE), + Buffer.from(JSON.stringify(manifest, null, 2) + '\n', 'utf8'), + ); + return manifest; +} + +/** + * Read and validate the sealed manifest from `staging`. + * @param {string} staging + * @returns {PrivateManifest} + */ +export function readPrivateManifest(staging) { + let value; + try { + value = JSON.parse( + readOrdinaryFile(join(staging, MANIFEST_FILE), LIMITS.recordBytes * 4).toString('utf8'), + ); + } catch { + refuse('UNREADABLE'); + } + return validateManifest(value); +} diff --git a/tools/cli-package/pack.mjs b/tools/cli-package/pack.mjs index 13f14ea0..7385427f 100644 --- a/tools/cli-package/pack.mjs +++ b/tools/cli-package/pack.mjs @@ -31,6 +31,7 @@ console.log( files: receipt.files.length, candidateFiles: identity.files, privateMaps: 'staged privately, then discarded', - symbolication: 'not implemented until Wave 3 Step 9a.2', + privateManifest: 'sealed privately, then discarded', + symbolication: 'actual packed CLI and core frames verified', }), ); diff --git a/tools/cli-package/prepack.test.mjs b/tools/cli-package/prepack.test.mjs index 56b5cabd..ee86acba 100644 --- a/tools/cli-package/prepack.test.mjs +++ b/tools/cli-package/prepack.test.mjs @@ -30,6 +30,7 @@ function fixture({ buildExit = 0, tsup: installed = true, finalizer } = {}) { for (const path of [ 'cli-package/prepack.mjs', 'cli-package/finalize.mjs', + 'cli-package/source-map.mjs', 'cli-package/files.mjs', 'cli-package/paths.mjs', 'bundle-closure/closure.mjs', diff --git a/tools/cli-package/produce.mjs b/tools/cli-package/produce.mjs index ece697e4..20c81642 100644 --- a/tools/cli-package/produce.mjs +++ b/tools/cli-package/produce.mjs @@ -115,9 +115,13 @@ export async function producePackage({ chmodSync(root, 0o700); const candidate = join(root, 'source'); const [privateStaging] = staging(); + // The symbolication run's home, project and captured frames: owner-only, outside the candidate. + const work = join(root, 'symbolication'); mkdirSync(candidate, { mode: 0o700 }); - mkdirSync(privateStaging, { mode: 0o700 }); - chmodSync(privateStaging, 0o700); + for (const directory of [privateStaging, work]) { + mkdirSync(directory, { mode: 0o700 }); + chmodSync(directory, 0o700); + } // Native real paths carry the letter case the filesystem stores, so no alias passes for elsewhere. const published = realpathSync.native(destination); const checkout = realpathSync.native(repository); @@ -148,6 +152,8 @@ export async function producePackage({ staging: privateStaging, source: identity.source, dirty: identity.dirty, + checkout: repository, + work, }); }, }); diff --git a/tools/cli-package/produce.test.mjs b/tools/cli-package/produce.test.mjs index d70c1045..933d73b4 100644 --- a/tools/cli-package/produce.test.mjs +++ b/tools/cli-package/produce.test.mjs @@ -84,6 +84,7 @@ function ports({ plantCache = false, failEmit = false, dirty = false } = {}) { }, seal: async (artifactDirectory, candidate) => { seen.seal = { artifactDirectory, ...candidate }; + seen.workMode = statSync(candidate.work).mode & 0o777; return { mode: 'compiled_package_test', source: candidate.source, files: [] }; }, }, @@ -154,13 +155,18 @@ test('the producer builds and seals only the owned candidate, never the original assert.equal(seen.emit.staging, join(candidate, '..', 'staging')); assert.equal(seen.noBuildCache, candidate); assert.deepEqual(seen.buildStateLeft, []); + // The seal symbolicates against the original checkout's objects, in a private work directory + // beside the candidate and its staging. assert.deepEqual(seen.seal, { artifactDirectory: destination, repository: candidate, staging: seen.emit.staging, source: SHA, dirty: false, + checkout: repository, + work: join(candidate, '..', 'symbolication'), }); + if (posixModes) assert.equal(seen.workMode, 0o700); if (posixModes) { assert.equal(seen.stagingMode, 0o700); assert.equal(seen.rootMode, 0o700); @@ -895,6 +901,17 @@ test('the producer entry point prints only the fixed failure lines and exits 1', assert.equal(result.stderr.includes(QUOTED_MAP), false); }); +test('a refused manifest or symbolication prints its own fixed code', async () => { + const { ManifestError } = await import('./manifest.mjs'); + const { SymbolicationError } = await import('./symbolicate.mjs'); + assert.deepEqual(productionFailure(new SymbolicationError('ANCHOR_UNMAPPED')), [ + '✗ CLI package production failed: PACKED_CLI_SYMBOLICATION_ANCHOR_UNMAPPED', + ]); + assert.deepEqual(productionFailure(new ManifestError('MAP_DRIFT')), [ + '✗ CLI package production failed: PACKED_CLI_MANIFEST_MAP_DRIFT', + ]); +}); + test('the failure report bounds what it forwards and forwards nothing but anchored diagnostics', () => { const failed = (stdout, stderr) => Object.assign(new Error('PACKED_CLI_CHILD_FAILED: 1'), { result: { stdout, stderr } }); @@ -962,7 +979,8 @@ test('the producer entry point hands the checkout, the artifact directory and it files: 2, candidateFiles: 3, privateMaps: 'staged privately, then discarded', - symbolication: 'not implemented until Wave 3 Step 9a.2', + privateManifest: 'sealed privately, then discarded', + symbolication: 'actual packed CLI and core frames verified', }); // Exactly one artifact directory: any other count refuses before the producer runs. rmSync(marker); diff --git a/tools/cli-package/seal.mjs b/tools/cli-package/seal.mjs index 2829b3d0..8b7fe98f 100644 --- a/tools/cli-package/seal.mjs +++ b/tools/cli-package/seal.mjs @@ -7,10 +7,12 @@ import assert from 'node:assert/strict'; import { createRequire } from 'node:module'; import { readFileSync, readdirSync, writeFileSync } from 'node:fs'; -import { basename, join } from 'node:path'; +import { basename, isAbsolute, join } from 'node:path'; import { inspectPackageArchive } from './archive.mjs'; import { originFromEnvironment, validatePackageReceipt } from './binding.mjs'; import { flatFiles, oneTarball, readOrdinaryFile, sha256 } from './files.mjs'; +import { sealPrivateManifest } from './manifest.mjs'; +import { symbolicateCandidate } from './symbolicate.mjs'; export function producerNodeMajor(root) { const declared = readOrdinaryFile(join(root, '.nvmrc'), 20).toString('utf8').trim(); @@ -92,13 +94,29 @@ export function finalizedPublicFiles(root, staging) { } /** + * Inspect the tarball, then seal the candidate's private map manifest, then symbolicate actual packed + * frames against it, and only then write the receipt: a refusal at any step leaves no receipt. * @param {string} artifactDirectory the public artifact directory holding exactly one tarball - * @param {{repository: string, staging: string, source: string, dirty: boolean}} candidate + * @param {{repository: string, staging: string, source: string, dirty: boolean, checkout: string, + * work: string, ports?: {manifest?: typeof sealPrivateManifest, + * symbolicate?: typeof symbolicateCandidate}}} candidate the candidate root and its private + * staging, the original checkout's identity and object store, and a private work directory */ -export async function sealPackage(artifactDirectory, { repository, staging, source, dirty }) { +export async function sealPackage( + artifactDirectory, + { repository, staging, source, dirty, checkout, work, ports = {} }, +) { + const { manifest = sealPrivateManifest, symbolicate = symbolicateCandidate } = ports; const tarball = oneTarball(artifactDirectory); assert.match(source, /^[0-9a-f]{40}$/u, 'PACKED_CLI_SOURCE_IDENTITY'); assert.equal(typeof dirty, 'boolean', 'PACKED_CLI_SOURCE_IDENTITY'); + assert.ok( + typeof checkout === 'string' && + isAbsolute(checkout) && + typeof work === 'string' && + isAbsolute(work), + 'PACKED_CLI_SEAL_INPUTS', + ); const archive = inspectPackageArchive(readOrdinaryFile(tarball, 16 * 1024 * 1024), { expectedFiles: finalizedPublicFiles(repository, staging), expectedPackage: expectedPackManifest(repository), @@ -106,6 +124,10 @@ export async function sealPackage(artifactDirectory, { repository, staging, sour const origin = originFromEnvironment(process.env); const pnpm = JSON.parse(readFileSync(join(repository, 'package.json'), 'utf8')).packageManager; assert.equal(pnpm, 'pnpm@9.12.3'); + // Only now, with the packed bytes proven to be the finalized ones, the private manifest is sealed; + // actual packed frames are then symbolicated against it before any receipt exists. + await manifest({ candidate: repository, staging, checkout, source, dirty, archive }); + await symbolicate({ candidate: repository, staging, checkout, work, archive }); const receipt = { version: 1, mode: 'compiled_package_test', diff --git a/tools/cli-package/seal.test.mjs b/tools/cli-package/seal.test.mjs index 6ebff51d..286deb9f 100644 --- a/tools/cli-package/seal.test.mjs +++ b/tools/cli-package/seal.test.mjs @@ -366,10 +366,26 @@ async function withCandidate(body, { packageManager = 'pnpm@9.12.3' } = {}) { const receiptPath = (artifact) => join(artifact, 'package-test-receipt.json'); +/** + * The seal with recording stand-ins for the private manifest and symbolication, whose own tests + * (symbolicate.test.mjs) run them for real. + */ +function sealWith(artifact, candidate, calls = []) { + return sealPackage(artifact, { + checkout: candidate.repository, + work: join(dirname(candidate.staging), 'work'), + ...candidate, + ports: { + manifest: async (options) => void calls.push(['manifest', options]), + symbolicate: async (options) => void calls.push(['symbolicate', options]), + }, + }); +} + test('the seal binds a locally produced candidate it is given, never the repository holding seal.mjs', async () => { await withCandidate(async ({ root, staging, artifact, tarball, files, published }) => { const receipt = await withOrigin({}, () => - sealPackage(artifact, { repository: root, staging, source: SOURCE, dirty: true }), + sealWith(artifact, { repository: root, staging, source: SOURCE, dirty: true }), ); assert.equal(receipt.mode, 'compiled_package_test'); assert.equal(receipt.source, SOURCE); @@ -395,11 +411,11 @@ test('the seal records the trusted job origin and refuses a dirty source there', await withCandidate(async ({ root, staging, artifact }) => { await withOrigin(GITHUB_ORIGIN, async () => { await assert.rejects( - sealPackage(artifact, { repository: root, staging, source: SOURCE, dirty: true }), + sealWith(artifact, { repository: root, staging, source: SOURCE, dirty: true }), /DIRTY_SOURCE/, ); assert.equal(existsSync(receiptPath(artifact)), false); - const receipt = await sealPackage(artifact, { + const receipt = await sealWith(artifact, { repository: root, staging, source: SOURCE, @@ -423,7 +439,7 @@ test('the seal refuses a candidate that pins a different pnpm, writing no receip async ({ root, staging, artifact }) => { await withOrigin({}, () => assert.rejects( - sealPackage(artifact, { repository: root, staging, source: SOURCE, dirty: false }), + sealWith(artifact, { repository: root, staging, source: SOURCE, dirty: false }), /pnpm@9\.12\.4/, ), ); @@ -442,7 +458,7 @@ test('the seal refuses an invalid source identity before it reads the candidate' { source: SOURCE, dirty: 'false' }, ]) await assert.rejects( - sealPackage(artifact, { repository: missing, staging, ...identity }), + sealWith(artifact, { repository: missing, staging, ...identity }), /PACKED_CLI_SOURCE_IDENTITY/, ); assert.equal(existsSync(receiptPath(artifact)), false); @@ -455,10 +471,108 @@ test('the seal refuses archive bytes that differ from the finalized candidate', writeFileSync(join(artifact, 'relavium-0.1.1.tgz'), gzipSync(ustar(files))); await withOrigin({}, () => assert.rejects( - sealPackage(artifact, { repository: root, staging, source: SOURCE, dirty: false }), + sealWith(artifact, { repository: root, staging, source: SOURCE, dirty: false }), /FILE_DRIFT/, ), ); assert.equal(existsSync(receiptPath(artifact)), false); }); }); + +test('the seal seals the private manifest after the inspection and symbolicates before any receipt', async () => { + await withCandidate(async ({ root, staging, artifact }) => { + const calls = []; + const work = join(dirname(staging), 'work'); + const receipt = await withOrigin({}, () => + sealWith(artifact, { repository: root, staging, source: SOURCE, dirty: false }, calls), + ); + assert.deepEqual( + calls.map(([step]) => step), + ['manifest', 'symbolicate'], + ); + const [[, manifest], [, symbolicate]] = calls; + // Both steps receive the inspected archive: the bytes the receipt then records. + assert.equal(manifest.archive.sha256, receipt.tarballSha256); + assert.deepEqual(manifest.archive.files, receipt.files); + assert.deepEqual( + { ...manifest, archive: undefined }, + { + candidate: root, + staging, + checkout: root, + source: SOURCE, + dirty: false, + archive: undefined, + }, + ); + assert.deepEqual( + { ...symbolicate, archive: undefined }, + { candidate: root, staging, checkout: root, work, archive: undefined }, + ); + assert.equal(symbolicate.archive, manifest.archive); + }); +}); + +test('a refused manifest or symbolication leaves no receipt, and an unproven archive reaches neither', async () => { + for (const failing of ['manifest', 'symbolicate']) { + await withCandidate(async ({ root, staging, artifact }) => { + const steps = []; + const refusal = new Error(`synthetic ${failing} refusal`); + const step = (name) => async () => { + steps.push(name); + if (name === failing) throw refusal; + }; + await withOrigin({}, () => + assert.rejects( + sealPackage(artifact, { + repository: root, + staging, + source: SOURCE, + dirty: false, + checkout: root, + work: join(dirname(staging), 'work'), + ports: { manifest: step('manifest'), symbolicate: step('symbolicate') }, + }), + (error) => error === refusal, + ), + ); + assert.deepEqual(steps, failing === 'manifest' ? ['manifest'] : ['manifest', 'symbolicate']); + assert.equal(existsSync(receiptPath(artifact)), false); + }); + } + await withCandidate(async ({ root, staging, artifact, files }) => { + files.set('package/README.md', Buffer.from('changed\n')); + writeFileSync(join(artifact, 'relavium-0.1.1.tgz'), gzipSync(ustar(files))); + const calls = []; + await withOrigin({}, () => + assert.rejects( + sealWith(artifact, { repository: root, staging, source: SOURCE, dirty: false }, calls), + /FILE_DRIFT/, + ), + ); + assert.deepEqual(calls, []); + }); +}); + +test('the seal requires the checkout and a private work directory before it reads the candidate', async () => { + await withCandidate(async ({ staging, artifact }) => { + const missing = join(artifact, 'no-candidate'); + for (const inputs of [ + { checkout: undefined, work: '/private/work' }, + { checkout: 'relative', work: '/private/work' }, + { checkout: '/checkout', work: undefined }, + { checkout: '/checkout', work: 'relative' }, + ]) { + await assert.rejects( + sealPackage(artifact, { + repository: missing, + staging, + source: SOURCE, + dirty: false, + ...inputs, + }), + /PACKED_CLI_SEAL_INPUTS/, + ); + } + }); +}); diff --git a/tools/cli-package/source-map.mjs b/tools/cli-package/source-map.mjs new file mode 100644 index 00000000..d7df73ad --- /dev/null +++ b/tools/cli-package/source-map.mjs @@ -0,0 +1,181 @@ +/** + * Strict Source Map v3 `mappings` decoding for the candidate finalizer and symbolication (ADR-0111, + * Step 9a.2a). Node's built-in `node:module` SourceMap is lenient — it decodes an invalid character as + * zero and extrapolates a position it has no segment for — so it serves only as an independent test + * oracle, never as the validator. A refusal names a fixed code and never quotes the mappings. + */ + +export class SourceMapError extends Error { + /** @param {string} code */ + constructor(code) { + super(`source map refused: ${code}`); + this.name = 'SourceMapError'; + this.code = code; + } +} + +/** @param {string} code @returns {never} */ +function refuse(code) { + throw new SourceMapError(code); +} + +const BASE64 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; +const DIGITS = new Int8Array(128).fill(-1); +for (let index = 0; index < BASE64.length; index += 1) DIGITS[BASE64.charCodeAt(index)] = index; +/** The largest unsigned VLQ value (a signed 32-bit magnitude and its sign bit). */ +const VLQ_MAX = 0xffffffff; + +/** + * One decoded segment: a generated column, and for a mapped segment its source index and 0-based + * original line and column, and optionally a name index. + * @typedef {{generatedColumn: number, source?: number, originalLine?: number, + * originalColumn?: number, name?: number}} Segment + */ + +/** + * Decode `mappings` into one array of segments per generated line. Refuses a character outside the + * base64 alphabet, a truncated or oversized value, an empty segment, a segment of other than 1, 4 or 5 + * fields, a generated column that does not increase along its line, and a negative or out-of-range + * source, name, line or column. + * @param {string} mappings + * @param {{sources: number, names: number}} counts the lengths of the map's `sources` and `names` + * @returns {Segment[][]} + */ +export function decodeMappings(mappings, { sources, names }) { + if (typeof mappings !== 'string') refuse('MAPPINGS_TYPE'); + /** @type {Segment[][]} */ + const lines = [[]]; + let column = 0; + let source = 0; + let originalLine = 0; + let originalColumn = 0; + let name = 0; + /** @type {number[]} */ + let fields = []; + let position = 0; + const finish = () => { + if (fields.length !== 1 && fields.length !== 4 && fields.length !== 5) + refuse('MAPPINGS_FIELDS'); + const line = lines[lines.length - 1]; + column += fields[0]; + if (column < 0) refuse('MAPPINGS_RANGE'); + if (line.length > 0 && column <= line[line.length - 1].generatedColumn) + refuse('MAPPINGS_ORDER'); + /** @type {Segment} */ + const segment = { generatedColumn: column }; + if (fields.length >= 4) { + source += fields[1]; + originalLine += fields[2]; + originalColumn += fields[3]; + if (source < 0 || source >= sources || originalLine < 0 || originalColumn < 0) { + refuse('MAPPINGS_RANGE'); + } + segment.source = source; + segment.originalLine = originalLine; + segment.originalColumn = originalColumn; + if (fields.length === 5) { + name += fields[4]; + if (name < 0 || name >= names) refuse('MAPPINGS_RANGE'); + segment.name = name; + } + } + line.push(segment); + fields = []; + }; + while (position < mappings.length) { + const character = mappings.charCodeAt(position); + if (character === 0x3b /* ; */ || character === 0x2c /* , */) { + // An empty segment is malformed; an empty line is not. + if (fields.length > 0) finish(); + else if (character === 0x2c || mappings.charCodeAt(position - 1) === 0x2c) { + refuse('MAPPINGS_EMPTY_SEGMENT'); + } + if (character === 0x3b) { + lines.push([]); + column = 0; + } + position += 1; + continue; + } + let value = 0; + let scale = 1; + for (;;) { + if (position >= mappings.length) refuse('MAPPINGS_TRUNCATED'); + const code = mappings.charCodeAt(position); + const digit = code < 128 ? DIGITS[code] : -1; + if (digit < 0) refuse('MAPPINGS_CHARACTER'); + position += 1; + value += (digit & 31) * scale; + if (value > VLQ_MAX) refuse('MAPPINGS_RANGE'); + if ((digit & 32) === 0) break; + scale *= 32; + } + const magnitude = Math.floor(value / 2); + fields.push(value % 2 === 1 ? -magnitude : magnitude); + } + if (fields.length > 0) finish(); + else if (mappings.charCodeAt(mappings.length - 1) === 0x2c) refuse('MAPPINGS_EMPTY_SEGMENT'); + return lines; +} + +/** + * The segment that starts exactly at a 0-based generated position, or `undefined`. Never the + * nearest preceding one: a frame that is off by a column must not map. + * @param {Segment[][]} lines @param {number} line @param {number} column + */ +export function segmentAt(lines, line, column) { + const segments = lines[line]; + if (segments === undefined) return undefined; + let low = 0; + let high = segments.length - 1; + while (low <= high) { + const middle = (low + high) >>> 1; + const start = segments[middle].generatedColumn; + if (start === column) return segments[middle]; + if (start < column) low = middle + 1; + else high = middle - 1; + } + return undefined; +} + +/** The line terminators ECMAScript and TypeScript both count. */ +const LINE_TERMINATOR = /\r\n|[\n\r\u2028\u2029]/u; + +/** + * The UTF-16 length of each line of `text`, as V8 and TypeScript count lines and columns. + * @param {string} text + */ +export function lineLengths(text) { + return text.split(LINE_TERMINATOR).map((line) => line.length); +} + +/** + * Require every generated position to lie inside the generated text: no segment on a line the text + * does not have, and no segment past its line's end (a position at the end is a caret position, as + * esbuild maps an empty line at column 0). + * @param {Segment[][]} lines @param {number[]} generated the generated text's line lengths + */ +export function assertGeneratedRange(lines, generated) { + for (let line = 0; line < lines.length; line += 1) { + const segments = lines[line]; + if (segments.length === 0) continue; + if (line >= generated.length) refuse('GENERATED_RANGE'); + if (segments[segments.length - 1].generatedColumn > generated[line]) refuse('GENERATED_RANGE'); + } +} + +/** + * Require every original position to lie inside its source text: an existing line, and a column no + * further than that line's end. + * @param {Segment[][]} lines @param {readonly number[][]} originals line lengths per source index + */ +export function assertOriginalRange(lines, originals) { + for (const segments of lines) { + for (const segment of segments) { + if (segment.source === undefined) continue; + const lengths = originals[segment.source]; + const length = lengths?.[segment.originalLine ?? -1]; + if (length === undefined || (segment.originalColumn ?? 0) > length) refuse('ORIGINAL_RANGE'); + } + } +} diff --git a/tools/cli-package/source-map.test.mjs b/tools/cli-package/source-map.test.mjs new file mode 100644 index 00000000..043554de --- /dev/null +++ b/tools/cli-package/source-map.test.mjs @@ -0,0 +1,279 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs'; +import { SourceMap, createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import { basename, join } from 'node:path'; +import { after, test } from 'node:test'; +import { pathToFileURL } from 'node:url'; +import { + SourceMapError, + assertGeneratedRange, + assertOriginalRange, + decodeMappings, + lineLengths, + segmentAt, +} from './source-map.mjs'; + +// The exact esbuild behind the installed tsup builds the fixtures; no product build runs here. +const cliRequire = createRequire(new URL('../../apps/cli/package.json', import.meta.url)); +const { build } = createRequire(cliRequire.resolve('tsup/package.json'))('esbuild'); +const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-source-map-'))); +after(() => rmSync(scratch, { recursive: true, force: true })); + +const BASE64 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; + +/** Encode one signed value as base64 VLQ, as a source map writer does. */ +function vlq(value) { + let rest = value < 0 ? -value * 2 + 1 : value * 2; + let out = ''; + do { + let digit = rest % 32; + rest = Math.floor(rest / 32); + if (rest > 0) digit += 32; + out += BASE64[digit]; + } while (rest > 0); + return out; +} + +/** Encode absolute segments ([column] or [column, source, line, column, name?]) per line. */ +function encode(lines) { + let source = 0; + let line = 0; + let column = 0; + let name = 0; + return lines + .map((segments) => { + let generated = 0; + return segments + .map((segment) => { + let out = vlq(segment[0] - generated); + generated = segment[0]; + if (segment.length >= 4) { + out += vlq(segment[1] - source) + vlq(segment[2] - line) + vlq(segment[3] - column); + [source, line, column] = [segment[1], segment[2], segment[3]]; + } + if (segment.length === 5) { + out += vlq(segment[4] - name); + name = segment[4]; + } + return out; + }) + .join(','); + }) + .join(';'); +} + +const refuses = (code) => (error) => error instanceof SourceMapError && error.code === code; +const COUNTS = { sources: 2, names: 2 }; + +test('decodes every segment exactly as Node’s own source map does', () => { + // A deterministic map with long lines, empty lines, unmapped and named segments and backward + // original moves, decoded strictly and compared with node:module at every segment start. + let seed = 7; + const next = (bound) => { + seed = (seed * 1103515245 + 12345) % 2 ** 31; + return seed % bound; + }; + const lines = []; + for (let line = 0; line < 40; line += 1) { + const segments = []; + let column = next(3); + const count = next(30); + for (let index = 0; index < count; index += 1) { + column += 1 + next(5000); + const kind = next(5); + if (kind === 0) segments.push([column]); + else if (kind === 1) segments.push([column, next(2), next(400), next(300)]); + else segments.push([column, next(2), next(400), next(300), next(2)]); + } + lines.push(segments); + } + const map = { version: 3, sources: ['a.ts', 'b.ts'], names: ['x', 'y'], mappings: encode(lines) }; + const decoded = decodeMappings(map.mappings, COUNTS); + const oracle = new SourceMap(map); + let checked = 0; + decoded.forEach((segments, line) => { + assert.equal(segments.length, lines[line].length); + segments.forEach((segment, index) => { + const expected = lines[line][index]; + assert.equal(segment.generatedColumn, expected[0]); + assert.equal(segmentAt(decoded, line, segment.generatedColumn), segment); + if (expected.length === 1) { + assert.equal(segment.source, undefined); + return; + } + const entry = oracle.findEntry(line, segment.generatedColumn); + assert.equal(entry.generatedLine, line); + assert.equal(entry.generatedColumn, segment.generatedColumn); + assert.equal(entry.originalSource, map.sources[segment.source]); + assert.equal(entry.originalLine, segment.originalLine); + assert.equal(entry.originalColumn, segment.originalColumn); + assert.equal(entry.name, segment.name === undefined ? undefined : map.names[segment.name]); + checked += 1; + }); + }); + assert.ok(checked > 200, 'the fixture exercises many mapped segments'); +}); + +test('maps a real esbuild frame below a hashbang and past astral characters to its original column', async () => { + const source = [ + '#!/usr/bin/env node', + "const label: string = '😀 ✓ — ü';", + 'const template = `one', + '', + 'three`;', + 'export function fail(reason: string): never {', + ' const note = reason;', + " void '😀😀'; throw new Error(note + label + template);", + '}', + "fail('x');", + '', + ].join('\n'); + writeFileSync(join(scratch, 'frame.ts'), source); + const built = await build({ + absWorkingDir: scratch, + entryPoints: ['frame.ts'], + outdir: 'out', + format: 'esm', + platform: 'node', + target: 'node22', + bundle: true, + minify: true, + sourcemap: 'external', + sourcesContent: false, + write: false, + logLevel: 'silent', + }); + const js = built.outputFiles.find((file) => file.path.endsWith('.js')).text; + const map = JSON.parse(built.outputFiles.find((file) => file.path.endsWith('.map')).text); + writeFileSync(join(scratch, 'frame.mjs'), js); + const decoded = decodeMappings(map.mappings, { + sources: map.sources.length, + names: map.names.length, + }); + // The hashbang stays on line 1 with no mapping, so code mappings start on line 2. + assert.match(js, /^#!\/usr\/bin\/env node\n/u); + assert.deepEqual(decoded[0], []); + assertGeneratedRange(decoded, lineLengths(js)); + assertOriginalRange(decoded, [lineLengths(source)]); + // Run it and read the throw site's 1-based line and column from V8's own structured call site. + const run = spawnSync( + process.execPath, + [ + '--input-type=module', + '--eval', + `Error.prepareStackTrace = (_, sites) => sites; +try { await import(${JSON.stringify(pathToFileURL(join(scratch, 'frame.mjs')).href)}); } +catch (error) { const site = error.stack.find((s) => s.getFileName()?.endsWith('frame.mjs')); + process.stdout.write(JSON.stringify([site.getLineNumber(), site.getColumnNumber()])); }`, + ], + { encoding: 'utf8', timeout: 20_000 }, + ); + assert.equal(run.status, 0, run.stderr); + const [line, column] = JSON.parse(run.stdout); + assert.ok(line >= 2); + const segment = segmentAt(decoded, line - 1, column - 1); + assert.ok(segment !== undefined, 'the throw site starts an exact segment'); + // `new` on source line 8, in UTF-16 code units: the two astral characters before it count twice. + assert.deepEqual( + [basename(map.sources[segment.source]), segment.originalLine + 1, segment.originalColumn + 1], + ['frame.ts', 8, 22], + ); + // One column off is no mapping at all, never the nearest preceding segment. + assert.equal(segmentAt(decoded, line - 1, column), undefined); +}); + +test('accepts empty maps, empty lines and unmapped segments', () => { + assert.deepEqual(decodeMappings('', COUNTS), [[]]); + assert.deepEqual(decodeMappings(';;', COUNTS), [[], [], []]); + assert.deepEqual(decodeMappings('A;C', COUNTS), [ + [{ generatedColumn: 0 }], + [{ generatedColumn: 1 }], + ]); + assert.deepEqual(decodeMappings('AAAA,CCCCC;', COUNTS), [ + [ + { generatedColumn: 0, source: 0, originalLine: 0, originalColumn: 0 }, + { generatedColumn: 1, source: 1, originalLine: 1, originalColumn: 1, name: 1 }, + ], + [], + ]); +}); + +test('refuses every malformed mapping with its own code', () => { + const largest = vlq(2 ** 31 - 1); + assert.equal(decodeMappings(largest, COUNTS)[0][0].generatedColumn, 2 ** 31 - 1); + for (const [mappings, code, counts = COUNTS] of [ + ['AAAA!', 'MAPPINGS_CHARACTER'], + ['AAéA', 'MAPPINGS_CHARACTER'], + ['A A', 'MAPPINGS_CHARACTER'], + ['g', 'MAPPINGS_TRUNCATED'], + ['AAAg', 'MAPPINGS_TRUNCATED'], + ['AA', 'MAPPINGS_FIELDS'], + ['AAA', 'MAPPINGS_FIELDS'], + ['AAAAAA', 'MAPPINGS_FIELDS'], + [',', 'MAPPINGS_EMPTY_SEGMENT'], + ['A,', 'MAPPINGS_EMPTY_SEGMENT'], + ['A,,A', 'MAPPINGS_EMPTY_SEGMENT'], + ['A,;A', 'MAPPINGS_EMPTY_SEGMENT'], + [';,A', 'MAPPINGS_EMPTY_SEGMENT'], + ['D', 'MAPPINGS_RANGE'], + ['ADAA', 'MAPPINGS_RANGE'], + ['AEAA', 'MAPPINGS_RANGE'], + ['AADA', 'MAPPINGS_RANGE'], + ['AAAD', 'MAPPINGS_RANGE'], + ['AAAAE', 'MAPPINGS_RANGE'], + ['AAAAD', 'MAPPINGS_RANGE'], + ['AAAA', 'MAPPINGS_RANGE', { sources: 0, names: 0 }], + [vlq(2 ** 32), 'MAPPINGS_RANGE'], + ['C,A', 'MAPPINGS_ORDER'], + ['E,D', 'MAPPINGS_ORDER'], + ]) { + assert.throws(() => decodeMappings(mappings, counts), refuses(code), mappings); + } + assert.throws(() => decodeMappings(undefined, COUNTS), refuses('MAPPINGS_TYPE')); + // A refusal never quotes the mappings it read. + assert.throws( + () => decodeMappings('secretAAAA!', COUNTS), + (error) => !error.message.includes('secret'), + ); +}); + +test('finds only the segment that starts exactly at a position', () => { + const lines = decodeMappings('A,EAAE,KAEC;;GAAA', COUNTS); + assert.equal(segmentAt(lines, 0, 0).generatedColumn, 0); + assert.equal(segmentAt(lines, 0, 2).originalColumn, 2); + assert.equal(segmentAt(lines, 0, 7).originalLine, 2); + for (const [line, column] of [ + [0, 1], + [0, 3], + [0, 8], + [1, 0], + [2, 0], + [2, 2], + [3, 0], + [-1, 0], + ]) { + assert.equal(segmentAt(lines, line, column), undefined, `${line}:${column}`); + } +}); + +test('counts lines and UTF-16 columns as V8 and TypeScript do', () => { + assert.deepEqual(lineLengths('ab\ncd\r\nefg\rh
😀
'), [2, 2, 3, 1, 2, 0]); + assert.deepEqual(lineLengths(''), [0]); +}); + +test('requires generated and original positions inside their texts', () => { + const lines = decodeMappings('A,EAAE;CACA', COUNTS); + assertGeneratedRange(lines, [2, 1]); + assert.throws(() => assertGeneratedRange(lines, [1, 1]), refuses('GENERATED_RANGE')); + assert.throws(() => assertGeneratedRange(lines, [2]), refuses('GENERATED_RANGE')); + // Original positions (0, 2) and (1, 2) of source 0; a column at a line's end is a caret position. + assertOriginalRange(lines, [[2, 2], []]); + assert.throws(() => assertOriginalRange(lines, [[2, 1], []]), refuses('ORIGINAL_RANGE')); + assert.throws(() => assertOriginalRange(lines, [[1, 2], []]), refuses('ORIGINAL_RANGE')); + assert.throws(() => assertOriginalRange(lines, [[2], []]), refuses('ORIGINAL_RANGE')); + assert.throws(() => assertOriginalRange(lines, [[], []]), refuses('ORIGINAL_RANGE')); + // Unmapped segments name no source and pass. + assertOriginalRange(decodeMappings('A,C', COUNTS), []); +}); diff --git a/tools/cli-package/symbolicate.mjs b/tools/cli-package/symbolicate.mjs new file mode 100644 index 00000000..47a432ac --- /dev/null +++ b/tools/cli-package/symbolicate.mjs @@ -0,0 +1,346 @@ +/** + * Symbolication of actual packed CLI and engine frames (ADR-0111; packaging.md "Candidate + * finalization — Step 9" item 5; Step 9a.2a). After the archive inspection and the private manifest, + * the producer runs the finalized `dist` — byte-equal to the tarball, re-hashed per frame — under the + * tooling-only preload (frame-capture.mjs) while it imports a canary workflow, and maps the captured + * frames through the staged final maps. Each anchor below must be reached by a synchronous frame that + * lands exactly on a mapping segment and resolves to the call site that the matching source + * revision's syntax tree gives: the candidate copy for a dirty local build, the checkout's Git objects + * at the recorded commit otherwise, both bound to the manifest's per-source digests. A refusal names a + * fixed code; no frame, path, position or source text is ever printed. + */ +import { Buffer } from 'node:buffer'; +import { randomBytes } from 'node:crypto'; +import { mkdirSync, realpathSync, writeFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { basename, dirname, join } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { readPrivateManifest } from './manifest.mjs'; +import { readOrdinaryFile, sha256 } from './files.mjs'; +import { isolatedEnvironment, isolatedGitEnvironment, runCommand } from './process.mjs'; +import { SourceMapError, decodeMappings, segmentAt } from './source-map.mjs'; + +export class SymbolicationError extends Error { + /** @param {string} code */ + constructor(code) { + super(`PACKED_CLI_SYMBOLICATION_${code}`); + this.name = 'SymbolicationError'; + this.code = code; + } +} + +/** @param {string} code @returns {never} */ +function refuse(code) { + throw new SymbolicationError(code); +} + +/** + * The named call sites the canary import passes through: three in the CLI and two in the inlined + * `@relavium/core`. Each is the one call to `call` inside the function `within`; its expected line and + * column come from the source revision, never from this table. + */ +export const ANCHORS = Object.freeze([ + { source: 'packages/core/src/yaml-decode.ts', within: 'decodeHardenedYaml', call: 'parseYaml' }, + { source: 'packages/core/src/parser.ts', within: 'parseWorkflow', call: 'decodeHardenedYaml' }, + { + source: 'apps/cli/src/authoring/authoring.ts', + within: 'parseAsWorkflow', + call: 'parseWorkflow', + }, + { + source: 'apps/cli/src/authoring/authoring.ts', + within: 'detectAndParse', + call: 'parseAsWorkflow', + }, + { source: 'apps/cli/src/commands/import.ts', within: 'importCommand', call: 'detectAndParse' }, +]); + +const PRELOAD = fileURLToPath(new URL('./frame-capture.mjs', import.meta.url)); +const FRAME_LIMIT = 64; + +/** A workflow the CLI accepts, whose id carries the canary the preload watches for. */ +function canaryDocument(canary) { + return [ + "schema_version: '1.0'", + 'workflow:', + ` id: ${canary}`, + ' name: Symbolication canary', + ' nodes:', + ' - { id: start, type: input }', + ' - { id: out, type: output }', + ' edges:', + ' - { from: start, to: out }', + '', + ].join('\n'); +} + +/** + * @typedef {{file: string | null, line: number, column: number, async: boolean}} Frame + */ + +/** + * Run the finalized CLI's `import` of a canary workflow under the preload, with its home, project and + * temporary files in `work` (owner-only, outside the candidate), and return the one captured stack. + * @param {{candidate: string, work: string, run?: typeof runCommand}} options + * @returns {Promise} + */ +export async function captureFrames({ candidate, work, run = runCommand }) { + const canary = `frames-canary-${randomBytes(6).toString('hex')}`; + const home = join(work, 'home'); + const project = join(work, 'project'); + for (const directory of [home, project]) mkdirSync(directory, { mode: 0o700 }); + const document = join(work, `${canary}.relavium.yaml`); + writeFileSync(document, canaryDocument(canary), { mode: 0o600, flag: 'wx' }); + const log = join(work, 'frames.json'); + try { + await run( + process.execPath, + [ + '--import', + pathToFileURL(PRELOAD).href, + join(candidate, 'apps', 'cli', 'dist', 'index.js'), + 'import', + document, + '--cwd', + project, + ], + { + cwd: work, + env: { + ...isolatedEnvironment(home), + RELAVIUM_FRAME_CAPTURE_LOG: log, + RELAVIUM_FRAME_CAPTURE_PACKAGE: join(candidate, 'apps', 'cli', 'package.json'), + RELAVIUM_FRAME_CAPTURE_CANARY: canary, + }, + timeoutMs: 120_000, + }, + ); + } catch { + // The child's own output is never printed: it can quote the candidate. + refuse('CAPTURE_FAILED'); + } + let record; + try { + record = JSON.parse(readOrdinaryFile(log, 1024 * 1024).toString('utf8')); + } catch { + refuse('CAPTURE_FAILED'); + } + if (record?.version !== 1 || !Array.isArray(record.captures)) refuse('CAPTURE_SHAPE'); + if (record.captures.length !== 1) refuse('CAPTURE_COUNT'); + const [frames] = record.captures; + if ( + !Array.isArray(frames) || + frames.length === 0 || + frames.length > FRAME_LIMIT || + !frames.every( + (frame) => + frame !== null && + typeof frame === 'object' && + (frame.file === null || typeof frame.file === 'string') && + typeof frame.async === 'boolean' && + (frame.file === null || + (Number.isSafeInteger(frame.line) && + frame.line > 0 && + Number.isSafeInteger(frame.column) && + frame.column > 0)), + ) + ) { + refuse('CAPTURE_SHAPE'); + } + return frames; +} + +/** + * The text of each path at the matching source revision, bound to the manifest's digest: the + * candidate copy for a dirty local build, otherwise the blob at the recorded commit, read from the + * original checkout's object store by Git without configuration, filters or replace refs. + * @param {{candidate: string, checkout: string, manifest: import('./manifest.mjs').PrivateManifest, + * paths: readonly string[], run?: typeof runCommand}} options + * @returns {Promise>} + */ +export async function readSourceRevision({ + candidate, + checkout, + manifest, + paths, + run = runCommand, +}) { + const texts = new Map(); + for (const path of paths) { + const digest = manifest.sources[path]; + if (digest === undefined) refuse('SOURCE_REVISION'); + let bytes; + try { + bytes = manifest.source.dirty + ? readOrdinaryFile(join(candidate, ...path.split('/')), 8 * 1024 * 1024) + : Buffer.from( + ( + await run( + 'git', + ['--no-replace-objects', 'cat-file', 'blob', `${manifest.source.commit}:${path}`], + { cwd: checkout, env: isolatedGitEnvironment(), timeoutMs: 60_000 }, + ) + ).stdout, + 'utf8', + ); + } catch { + refuse('SOURCE_REVISION'); + } + if (sha256(bytes) !== digest) refuse('SOURCE_REVISION'); + texts.set(path, bytes.toString('utf8')); + } + return texts; +} + +/** + * The 1-based line and UTF-16 column of an anchor's call site in `text`: the start of the callee's + * name in the one call to `call` inside the one function named `within`. + * @param {typeof import('typescript')} ts @param {string} text + * @param {{source: string, within: string, call: string}} anchor + */ +export function anchorPosition(ts, text, anchor) { + const file = ts.createSourceFile( + anchor.source, + text, + ts.ScriptTarget.Latest, + true, + anchor.source.endsWith('.tsx') ? ts.ScriptKind.TSX : ts.ScriptKind.TS, + ); + const owners = []; + const visit = (node) => { + if ( + (ts.isFunctionDeclaration(node) || ts.isMethodDeclaration(node)) && + node.name !== undefined && + node.name.getText(file) === anchor.within + ) { + owners.push(node); + } else if ( + ts.isVariableDeclaration(node) && + ts.isIdentifier(node.name) && + node.name.text === anchor.within && + node.initializer !== undefined && + (ts.isArrowFunction(node.initializer) || ts.isFunctionExpression(node.initializer)) + ) { + owners.push(node.initializer); + } + ts.forEachChild(node, visit); + }; + visit(file); + if (owners.length !== 1) refuse('ANCHOR_SOURCE'); + const callees = []; + const find = (node) => { + if (ts.isCallExpression(node)) { + const callee = node.expression; + if (ts.isIdentifier(callee) && callee.text === anchor.call) callees.push(callee); + else if (ts.isPropertyAccessExpression(callee) && callee.name.text === anchor.call) { + callees.push(callee.name); + } + } + ts.forEachChild(node, find); + }; + find(owners[0]); + if (callees.length !== 1) refuse('ANCHOR_SOURCE'); + const { line, character } = file.getLineAndCharacterOfPosition(callees[0].getStart(file)); + return { line: line + 1, column: character + 1 }; +} + +/** + * Map the captured frames and require every anchor, against the manifest sealed into `staging`. Each + * frame inside the finalized `dist` must be the very file the manifest and the tarball record; its + * map must be the staged bytes the manifest records and decode strictly; only an exact segment counts. + * @param {{candidate: string, staging: string, checkout: string, work: string, + * archive: {files: {path: string, sha256: string}[]}, run?: typeof runCommand, + * capture?: typeof captureFrames, revision?: typeof readSourceRevision, + * anchors?: readonly {source: string, within: string, call: string}[]}} options + * @returns {Promise<{anchors: number}>} + */ +export async function symbolicateCandidate({ + candidate, + staging, + checkout, + work, + archive, + run = runCommand, + capture = captureFrames, + revision = readSourceRevision, + anchors = ANCHORS, +}) { + if (anchors.length === 0) refuse('ANCHORS'); + const manifest = readPrivateManifest(staging); + const frames = await capture({ candidate, work, run }); + const paths = [...new Set(anchors.map((anchor) => anchor.source))]; + const texts = await revision({ candidate, checkout, manifest, paths, run }); + let ts; + try { + ts = createRequire(join(candidate, 'apps', 'cli', 'package.json'))('typescript'); + } catch { + refuse('TOOLCHAIN'); + } + const outputs = new Map(manifest.outputs.map((output) => [output.path, output])); + const packed = new Map(archive.files.map((file) => [file.path, file.sha256])); + const dist = realpathSync.native(join(candidate, 'apps', 'cli', 'dist')); + const decoded = new Map(); + const mapsOf = (output) => { + let entry = decoded.get(output.path); + if (entry !== undefined) return entry; + const bytes = readOrdinaryFile( + join(staging, 'maps', `${basename(output.path)}.map`), + 16 * 1024 * 1024, + ); + if (sha256(bytes) !== output.map.sha256) refuse('MAP_DIGEST'); + try { + const map = JSON.parse(bytes.toString('utf8')); + entry = { + sources: map.sources, + lines: decodeMappings(map.mappings, { + sources: map.sources.length, + names: map.names.length, + }), + }; + } catch (error) { + if (error instanceof SourceMapError || error instanceof SyntaxError) refuse('MAP_DECODE'); + throw error; + } + decoded.set(output.path, entry); + return entry; + }; + const reached = []; + for (const frame of frames) { + if (frame.async || frame.file === null || !frame.file.startsWith('file:')) continue; + let path; + try { + path = realpathSync.native(fileURLToPath(frame.file)); + } catch { + refuse('FRAME_FILE'); + } + // Frames outside the finalized dist (the preload, yaml, commander) carry no anchor. + if (dirname(path) !== dist) continue; + const key = `dist/${basename(path)}`; + const output = outputs.get(key); + if (output === undefined) refuse('FRAME_FILE'); + const digest = sha256(readOrdinaryFile(path, 16 * 1024 * 1024)); + if (digest !== output.sha256 || packed.get(`package/${key}`) !== digest) refuse('FRAME_FILE'); + const { sources, lines } = mapsOf(output); + const segment = segmentAt(lines, frame.line - 1, frame.column - 1); + if (segment?.source === undefined) continue; + reached.push({ + source: sources[segment.source], + line: (segment.originalLine ?? 0) + 1, + column: (segment.originalColumn ?? 0) + 1, + }); + } + for (const anchor of anchors) { + const expected = anchorPosition(ts, texts.get(anchor.source), anchor); + if ( + !reached.some( + (frame) => + frame.source === anchor.source && + frame.line === expected.line && + frame.column === expected.column, + ) + ) { + refuse('ANCHOR_UNMAPPED'); + } + } + return { anchors: anchors.length }; +} diff --git a/tools/cli-package/symbolicate.test.mjs b/tools/cli-package/symbolicate.test.mjs new file mode 100644 index 00000000..1e8cc604 --- /dev/null +++ b/tools/cli-package/symbolicate.test.mjs @@ -0,0 +1,634 @@ +import assert from 'node:assert/strict'; +import { Buffer } from 'node:buffer'; +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + realpathSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import { dirname, join, posix } from 'node:path'; +import { after, test } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { MANIFEST_INPUTS, ManifestError, sealPrivateManifest } from './manifest.mjs'; +import { mapTrailer } from './finalize.mjs'; +import { + ANCHORS, + SymbolicationError, + anchorPosition, + captureFrames, + readSourceRevision, + symbolicateCandidate, +} from './symbolicate.mjs'; + +// A synthetic candidate whose TypeScript sits at the anchors' real repository paths, with the same +// function names, bundled by the exact esbuild behind the installed tsup with splitting and minify +// as the CLI build does. It runs under the real preload, so capture, maps and anchors are all real. +const cliRequire = createRequire(new URL('../../apps/cli/package.json', import.meta.url)); +const { build } = createRequire(cliRequire.resolve('tsup/package.json'))('esbuild'); +const ts = cliRequire('typescript'); +const cliModules = fileURLToPath(new URL('../../apps/cli/node_modules', import.meta.url)); +const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-symbolicate-'))); +after(() => rmSync(scratch, { recursive: true, force: true })); +const digest = (bytes) => createHash('sha256').update(bytes).digest('hex'); +const posixModes = process.platform !== 'win32'; +let count = 0; + +const SOURCES = { + 'packages/core/src/yaml-decode.ts': [ + "import { parse as parseYaml } from 'yaml';", + '', + 'export function decodeHardenedYaml(text: string): unknown {', + " return parseYaml(text, { version: '1.2' });", + '}', + '', + ], + 'packages/core/src/parser.ts': [ + "import { decodeHardenedYaml } from './yaml-decode.js';", + '', + 'export function parseWorkflow(text: string): { id: string } {', + ' const raw = decodeHardenedYaml(text) as { workflow: { id: string } };', + ' return { id: raw.workflow.id };', + '}', + '', + ], + 'apps/cli/src/authoring/authoring.ts': [ + "import { parseWorkflow } from '../../../../packages/core/src/parser.js';", + '', + 'export function detectAndParse(text: string, fileName: string): { id: string } {', + " if (fileName.endsWith('.relavium.yaml')) {", + ' return parseAsWorkflow(text);', + ' }', + " throw new Error('unsupported');", + '}', + '', + 'function parseAsWorkflow(text: string): { id: string } {', + ' return parseWorkflow(text);', + '}', + '', + ], + 'apps/cli/src/commands/import.ts': [ + "import { readFileSync } from 'node:fs';", + "import { basename } from 'node:path';", + "import { detectAndParse } from '../authoring/authoring.js';", + '', + 'export function importCommand(path: string): string {', + " const text = readFileSync(path, 'utf8');", + ' return detectAndParse(text, basename(path)).id;', + '}', + '', + ], + 'apps/cli/src/index.ts': [ + '#!/usr/bin/env node', + "import { Entry } from '@napi-rs/keyring';", + 'try {', + " new Entry('relavium', 'probe');", + '} catch {', + ' // The preload replaces the keyring binding with one that refuses.', + '}', + 'const [command, path] = process.argv.slice(2);', + "if (command === 'import' && path !== undefined) {", + " const { importCommand } = await import('./commands/import.js');", + ' importCommand(path);', + '}', + '', + ], +}; + +/** Write `path` (repository-relative) under `root`. */ +function put(root, path, bytes) { + mkdirSync(dirname(join(root, ...path.split('/'))), { recursive: true }); + writeFileSync(join(root, ...path.split('/')), bytes); +} + +/** + * A finalized synthetic candidate: its finalized dist without trailers, its staged maps with + * repository-relative sources, the finalizer's record, the manifest inputs and the CLI installation. + */ +async function candidate({ sources = SOURCES } = {}) { + count += 1; + const root = join(scratch, `candidate-${count}`); + const staging = join(scratch, `staging-${count}`); + const work = join(scratch, `work-${count}`); + for (const directory of [staging, work]) { + mkdirSync(directory); + chmodSync(directory, 0o700); + } + mkdirSync(join(staging, 'maps')); + for (const [path, lines] of Object.entries(sources)) put(root, path, lines.join('\n')); + for (const path of MANIFEST_INPUTS) { + put( + root, + path, + path.endsWith('.json') ? `${JSON.stringify({ synthetic: path })}\n` : `# ${path}\n`, + ); + } + put(root, 'apps/cli/tsup.config.ts', "export default {\n target: 'node22',\n};\n"); + put(root, 'apps/cli/package.json', JSON.stringify({ name: 'relavium', type: 'module' })); + symlinkSync(cliModules, join(root, 'apps', 'cli', 'node_modules'), 'junction'); + const cli = join(root, 'apps', 'cli'); + const built = await build({ + absWorkingDir: cli, + entryPoints: { index: 'src/index.ts' }, + outdir: 'dist', + format: 'esm', + platform: 'node', + target: 'node22', + bundle: true, + splitting: true, + minify: true, + sourcemap: 'external', + sourcesContent: false, + external: ['yaml', '@napi-rs/keyring'], + write: false, + logLevel: 'silent', + }); + const outputs = []; + const maps = []; + for (const file of built.outputFiles) { + const name = posix.basename(file.path.split('\\').join('/')); + if (name.endsWith('.map')) { + const map = JSON.parse(file.text); + map.sources = map.sources.map((source) => + posix.normalize(posix.join('apps/cli/dist', source)), + ); + const bytes = Buffer.from(JSON.stringify(map)); + writeFileSync(join(staging, 'maps', name), bytes); + maps.push({ path: `dist/${name}`, bytes: bytes.length, sha256: digest(bytes) }); + } else { + // An external map leaves no link in the code: this is already the finalized JS. + const code = file.text; + assert.equal(code.includes(mapTrailer(name)), false); + put(cli, `dist/${name}`, code); + const bytes = Buffer.from(code); + outputs.push({ path: `dist/${name}`, bytes: bytes.length, sha256: digest(bytes) }); + } + } + writeFileSync( + join(staging, 'finalized.json'), + JSON.stringify({ version: 1, outputs, maps, upstreamMaps: 0 }), + ); + const archive = { + sha256: 'a'.repeat(64), + package: { version: '0.1.1' }, + files: outputs.map((output) => ({ path: `package/${output.path}`, sha256: output.sha256 })), + }; + return { root, cli, staging, work, archive, outputs }; +} + +const TREE = 'b'.repeat(40); +const COMMIT = 'c'.repeat(40); +/** Git as the manifest asks it: the commit's tree. */ +const git = async (command, args) => { + assert.equal(command, 'git'); + assert.deepEqual(args, ['--no-replace-objects', 'rev-parse', '--verify', `${COMMIT}^{tree}`]); + return { stdout: `${TREE}\n`, stderr: '' }; +}; + +/** Seal the manifest of a dirty local candidate, whose source revision is its own copy. */ +async function seal(fixture) { + return sealPrivateManifest({ + candidate: fixture.root, + staging: fixture.staging, + checkout: fixture.root, + source: COMMIT, + dirty: true, + archive: fixture.archive, + run: git, + }); +} + +/** Re-record the finalized digests after a deliberate change, then seal a fresh manifest. */ +async function reseal(fixture) { + const outputs = readdirSync(join(fixture.cli, 'dist')).map((name) => { + const bytes = readFileSync(join(fixture.cli, 'dist', name)); + return { path: `dist/${name}`, bytes: bytes.length, sha256: digest(bytes) }; + }); + const maps = readdirSync(join(fixture.staging, 'maps')).map((name) => { + const bytes = readFileSync(join(fixture.staging, 'maps', name)); + return { path: `dist/${name}`, bytes: bytes.length, sha256: digest(bytes) }; + }); + rmSync(join(fixture.staging, 'finalized.json')); + rmSync(join(fixture.staging, 'manifest.json'), { force: true }); + writeFileSync( + join(fixture.staging, 'finalized.json'), + JSON.stringify({ version: 1, outputs, maps, upstreamMaps: 0 }), + ); + fixture.archive.files = outputs.map((output) => ({ + path: `package/${output.path}`, + sha256: output.sha256, + })); + return seal(fixture); +} + +function symbolicate(fixture, options = {}) { + return symbolicateCandidate({ + candidate: fixture.root, + staging: fixture.staging, + checkout: fixture.root, + work: fixture.work, + archive: fixture.archive, + ...options, + }); +} + +const refuses = (code) => (error) => error instanceof SymbolicationError && error.code === code; + +/** The finalized chunk that holds the core call sites. */ +function coreChunk(fixture) { + return readdirSync(join(fixture.cli, 'dist')).find((name) => + JSON.parse(readFileSync(join(fixture.staging, 'maps', `${name}.map`), 'utf8')).sources.includes( + 'packages/core/src/parser.ts', + ), + ); +} + +test('the manifest binds the commit, toolchain, inputs, finalized pairs, sources and tarball', async () => { + const fixture = await candidate(); + const manifest = await seal(fixture); + assert.deepEqual(manifest.source, { commit: COMMIT, tree: TREE, dirty: true }); + assert.equal(manifest.packageVersion, '0.1.1'); + assert.equal(manifest.target, 'node22'); + assert.equal(manifest.tarballSha256, 'a'.repeat(64)); + assert.equal(manifest.toolchain.node, process.version); + assert.equal(manifest.toolchain.typescript, ts.version); + assert.deepEqual(Object.keys(manifest.inputs), [...MANIFEST_INPUTS]); + assert.equal( + manifest.inputs['apps/cli/tsup.config.ts'], + digest(readFileSync(join(fixture.cli, 'tsup.config.ts'))), + ); + assert.deepEqual( + manifest.outputs.map((output) => output.path), + fixture.outputs.map((output) => output.path).sort(), + ); + for (const output of manifest.outputs) { + assert.equal(output.sha256, digest(readFileSync(join(fixture.cli, output.path)))); + assert.deepEqual(output.map, { + path: `${output.path}.map`, + sha256: digest( + readFileSync(join(fixture.staging, 'maps', `${posix.basename(output.path)}.map`)), + ), + }); + } + assert.deepEqual(Object.keys(manifest.sources), Object.keys(SOURCES).sort()); + for (const [path, value] of Object.entries(manifest.sources)) { + assert.equal(value, digest(readFileSync(join(fixture.root, ...path.split('/'))))); + } + // Written once, owner-only, and holding no absolute path. + const text = readFileSync(join(fixture.staging, 'manifest.json'), 'utf8'); + assert.deepEqual(JSON.parse(text), manifest); + assert.equal(text.includes(scratch), false); + if (posixModes) { + assert.equal(statSync(join(fixture.staging, 'manifest.json')).mode & 0o777, 0o600); + } + await assert.rejects(seal(fixture), (error) => error.code === 'EEXIST'); +}); + +test('the manifest refuses drifted maps and tarball entries and a missing tree', async () => { + for (const [label, change, code] of [ + [ + 'a staged map changed after the finalizer recorded it', + (fixture) => + writeFileSync(join(fixture.staging, 'maps', `${coreChunk(fixture)}.map`), '{"sources":[]}'), + 'MAP_DRIFT', + ], + [ + 'a tarball entry that is not the finalized JS', + (fixture) => (fixture.archive.files[0].sha256 = 'd'.repeat(64)), + 'TARBALL_DRIFT', + ], + [ + 'an output without its map in the record', + (fixture) => { + const record = JSON.parse(readFileSync(join(fixture.staging, 'finalized.json'), 'utf8')); + record.maps.pop(); + writeFileSync(join(fixture.staging, 'finalized.json'), JSON.stringify(record)); + }, + 'RECORD', + ], + [ + 'a build configuration without its target', + (fixture) => writeFileSync(join(fixture.cli, 'tsup.config.ts'), 'export default {};\n'), + 'TARGET', + ], + [ + 'a missing manifest input', + (fixture) => rmSync(join(fixture.root, 'pnpm-lock.yaml')), + 'UNREADABLE_INPUT', + ], + ]) { + const fixture = await candidate(); + change(fixture); + await assert.rejects( + seal(fixture), + (error) => error instanceof ManifestError && error.code === code, + label, + ); + assert.equal(existsSync(join(fixture.staging, 'manifest.json')), false, label); + } + const fixture = await candidate(); + await assert.rejects( + sealPrivateManifest({ + candidate: fixture.root, + staging: fixture.staging, + checkout: fixture.root, + source: COMMIT, + dirty: false, + archive: fixture.archive, + run: async () => ({ stdout: 'not a tree\n', stderr: '' }), + }), + (error) => error instanceof ManifestError && error.code === 'SOURCE_TREE', + ); +}); + +test('actual packed frames reach every anchor through the real preload and exact segments', async () => { + const fixture = await candidate(); + await seal(fixture); + let frames; + const result = await symbolicate(fixture, { + capture: async (options) => (frames = await captureFrames(options)), + }); + assert.deepEqual(result, { anchors: ANCHORS.length }); + // One stack crossed the entry, a lazy chunk and the inlined core. + const files = new Set( + frames + .filter((frame) => frame.file?.includes('/dist/') || frame.file?.includes('\\dist\\')) + .map((frame) => frame.file), + ); + assert.ok(files.size >= 2, 'the stack crosses more than one finalized chunk'); + // The run kept its files in the private work directory. + assert.equal(existsSync(join(fixture.work, 'frames.json')), true); + assert.equal(existsSync(join(fixture.root, '.relavium')), false); +}); + +test('the expected positions come from the source revision, independently of any map', () => { + for (const anchor of ANCHORS) { + const text = SOURCES[anchor.source].join('\n'); + const position = anchorPosition(ts, text, anchor); + const line = text.split('\n')[position.line - 1]; + assert.equal(line.slice(position.column - 1).startsWith(anchor.call), true, anchor.call); + } + // A member call is positioned at its property name; a second call or owner is ambiguous. + assert.deepEqual( + anchorPosition(ts, 'function f() {\n yaml.parse(x);\n}\n', { + source: 'a.ts', + within: 'f', + call: 'parse', + }), + { line: 2, column: 8 }, + ); + for (const text of [ + 'function f() { g(); g(); }', + 'function f() { h(); }', + 'function f() { g(); }\nfunction f() { g(); }', + ]) { + assert.throws( + () => anchorPosition(ts, text, { source: 'a.ts', within: 'f', call: 'g' }), + refuses('ANCHOR_SOURCE'), + text, + ); + } +}); + +test('wrong JS, a wrong map, permuted sources and a wrong source revision cannot reach the anchors', async () => { + // Every digest is re-recorded after each change, so only the mapping layer can refuse it. + for (const [label, change] of [ + [ + 'JS shifted by one column', + (fixture) => { + const path = join(fixture.cli, 'dist', coreChunk(fixture)); + writeFileSync(path, readFileSync(path, 'utf8').replace(/^/gmu, ' ')); + }, + ], + [ + 'JS shifted by one line', + (fixture) => { + const path = join(fixture.cli, 'dist', coreChunk(fixture)); + writeFileSync(path, `\n${readFileSync(path, 'utf8')}`); + }, + ], + [ + 'a map from a build one line apart', + (fixture) => { + const path = join(fixture.staging, 'maps', `${coreChunk(fixture)}.map`); + const map = JSON.parse(readFileSync(path, 'utf8')); + map.mappings = `;${map.mappings}`; + writeFileSync(path, JSON.stringify(map)); + }, + ], + [ + 'a map whose sources are permuted', + (fixture) => { + const path = join(fixture.staging, 'maps', `${coreChunk(fixture)}.map`); + const map = JSON.parse(readFileSync(path, 'utf8')); + map.sources.reverse(); + writeFileSync(path, JSON.stringify(map)); + }, + ], + [ + 'a source revision with one more line', + (fixture) => { + const path = join(fixture.root, 'packages', 'core', 'src', 'parser.ts'); + writeFileSync(path, `// shifted\n${readFileSync(path, 'utf8')}`); + }, + ], + ]) { + const fixture = await candidate(); + change(fixture); + await reseal(fixture); + await assert.rejects(symbolicate(fixture), refuses('ANCHOR_UNMAPPED'), label); + } + // Another chunk's map names other sources, so the resealed manifest no longer covers the anchors. + const fixture = await candidate(); + const core = `${coreChunk(fixture)}.map`; + writeFileSync( + join(fixture.staging, 'maps', core), + readFileSync(join(fixture.staging, 'maps', 'index.js.map')), + ); + await reseal(fixture); + await assert.rejects(symbolicate(fixture), refuses('SOURCE_REVISION')); +}); + +test('files that are not the manifest’s refuse by their digest', async () => { + for (const [label, change, code] of [ + [ + 'finalized JS changed after the manifest', + (fixture) => { + const path = join(fixture.cli, 'dist', coreChunk(fixture)); + writeFileSync(path, `${readFileSync(path, 'utf8')} `); + }, + 'FRAME_FILE', + ], + [ + 'a staged map changed after the manifest', + (fixture) => { + const path = join(fixture.staging, 'maps', `${coreChunk(fixture)}.map`); + writeFileSync(path, `${readFileSync(path, 'utf8')} `); + }, + 'MAP_DIGEST', + ], + [ + 'a source changed after the manifest', + (fixture) => { + const path = join(fixture.root, 'apps', 'cli', 'src', 'commands', 'import.ts'); + writeFileSync(path, `${readFileSync(path, 'utf8')}\n`); + }, + 'SOURCE_REVISION', + ], + [ + 'a tarball entry that is not the run file', + (fixture) => { + const key = `package/dist/${coreChunk(fixture)}`; + fixture.archive.files.find((file) => file.path === key).sha256 = 'e'.repeat(64); + }, + 'FRAME_FILE', + ], + ]) { + const fixture = await candidate(); + await seal(fixture); + change(fixture); + await assert.rejects(symbolicate(fixture), refuses(code), label); + } + const fixture = await candidate(); + await assert.rejects(symbolicate(fixture), (error) => error instanceof ManifestError); + await seal(fixture); + await assert.rejects(symbolicate(fixture, { anchors: [] }), refuses('ANCHORS')); +}); + +test('a clean candidate reads its source revision from the checkout’s Git objects', async () => { + const checkout = join(scratch, 'checkout'); + mkdirSync(checkout); + const gitRun = (args) => { + const run = spawnSync('git', args, { + cwd: checkout, + encoding: 'utf8', + env: { PATH: process.env.PATH, HOME: checkout, GIT_CONFIG_NOSYSTEM: '1' }, + }); + assert.equal(run.status, 0, run.stderr); + return run.stdout.trim(); + }; + gitRun(['init', '--quiet']); + const path = 'apps/cli/src/commands/import.ts'; + put(checkout, path, SOURCES[path].join('\n')); + gitRun(['add', '.']); + gitRun([ + '-c', + 'user.name=t', + '-c', + 'user.email=t@example.invalid', + 'commit', + '--quiet', + '-m', + 'x', + ]); + const commit = gitRun(['rev-parse', 'HEAD']); + const text = SOURCES[path].join('\n'); + const manifest = { source: { commit, dirty: false }, sources: { [path]: digest(text) } }; + const texts = await readSourceRevision({ + candidate: join(scratch, 'no-candidate'), + checkout, + manifest, + paths: [path], + }); + assert.equal(texts.get(path), text); + // A commit whose blob is not the built source refuses; so does a path outside the manifest. + await assert.rejects( + readSourceRevision({ + candidate: checkout, + checkout, + manifest: { ...manifest, sources: { [path]: digest(`${text}\n`) } }, + paths: [path], + }), + refuses('SOURCE_REVISION'), + ); + await assert.rejects( + readSourceRevision({ candidate: checkout, checkout, manifest, paths: ['apps/cli/src/x.ts'] }), + refuses('SOURCE_REVISION'), + ); +}); + +test('the capture refuses a run that fails, never parses the canary or parses it twice', async () => { + for (const [label, entry, code] of [ + ['a failed run', 'process.exit(3);', 'CAPTURE_FAILED'], + ['no canary parse', "import('yaml').then((y) => y.parse('a: 1'));", 'CAPTURE_COUNT'], + [ + 'two canary parses', + "import { readFileSync } from 'node:fs'; import { parse } from 'yaml';" + + "const t = readFileSync(process.argv[3], 'utf8'); parse(t); parse(t);", + 'CAPTURE_COUNT', + ], + ]) { + const fixture = await candidate(); + writeFileSync(join(fixture.cli, 'dist', 'index.js'), entry); + await assert.rejects( + captureFrames({ candidate: fixture.root, work: fixture.work }), + refuses(code), + label, + ); + } +}); + +test('the preload refuses the keyring and leaves V8’s stack formatting as it found it', async () => { + const fixture = await candidate(); + const canary = 'frames-canary-preload'; + const document = join(fixture.work, `${canary}.relavium.yaml`); + writeFileSync(document, `workflow:\n id: ${canary}\n`); + writeFileSync( + join(fixture.cli, 'dist', 'index.js'), + "import { readFileSync } from 'node:fs';\n" + + "import { Entry } from '@napi-rs/keyring';\n" + + "import { parse } from 'yaml';\n" + + 'let keyring;\n' + + "try { new Entry('relavium', 'probe'); keyring = 'native'; } catch (error) { keyring = error.message; }\n" + + "parse(readFileSync(process.argv[2], 'utf8'));\n" + + // A formatter left installed would make `stack` an array of call sites, not a string. + 'process.stdout.write(JSON.stringify({ keyring, limit: Error.stackTraceLimit,' + + " stack: typeof new Error('x').stack }));\n", + ); + const run = spawnSync( + process.execPath, + [ + '--import', + new URL('frame-capture.mjs', import.meta.url).href, + join(fixture.cli, 'dist', 'index.js'), + document, + ], + { + encoding: 'utf8', + timeout: 30_000, + env: { + PATH: process.env.PATH, + RELAVIUM_FRAME_CAPTURE_LOG: join(fixture.work, 'frames.json'), + RELAVIUM_FRAME_CAPTURE_PACKAGE: join(fixture.cli, 'package.json'), + RELAVIUM_FRAME_CAPTURE_CANARY: canary, + }, + }, + ); + assert.equal(run.status, 0, run.stderr); + assert.deepEqual(JSON.parse(run.stdout), { + keyring: 'The symbolication run refuses the keyring', + limit: 10, + stack: 'string', + }); + const record = JSON.parse(readFileSync(join(fixture.work, 'frames.json'), 'utf8')); + assert.equal(record.captures.length, 1); + // Owner-only, and never overwritten. + if (posixModes) assert.equal(statSync(join(fixture.work, 'frames.json')).mode & 0o777, 0o600); + // Without its configuration the preload refuses to run at all. + const bare = spawnSync( + process.execPath, + ['--import', new URL('frame-capture.mjs', import.meta.url).href, '--eval', ''], + { encoding: 'utf8', timeout: 30_000, env: { PATH: process.env.PATH } }, + ); + assert.notEqual(bare.status, 0); +}); From fd6cd1cdf1db19923dc18fbdd9a08cb295ace386 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sat, 24 Oct 2026 11:22:37 +0300 Subject: [PATCH 39/97] fix(repo): close the Step 9a.2a internal rigor pass The read-only rigor pass over 1ab4c134 (claims, platform, security, integration and critic lenses, and three mutation sweeps) found two high findings and several medium ones. All are corrected here. - A clean checkout with converted line endings (core.autocrlf, the Git for Windows default) refused every clean production: status stays clean while the copy holds CRLF bytes. Candidate preparation now confirms a clean status byte for byte (copyMatchesCommit compares each copy's Git blob id with the commit tree under isolated Git) and records a differing copy as dirty. - `relavium import --cwd` walks up for a project, so an ancestor .relavium/ (a developer's real project on Windows) could receive the canary. The capture creates the project's own .relavium/ first and refuses CAPTURE_PROJECT unless the canary landed there. - The manifest inputs include the configurations the engine build configurations extend. The two-core, three-CLI anchor ruling is enforced. An eighth VLQ digit refuses, because zero padding decoded as NaN and passed every comparison. Bodiless overloads are not anchor owners. The preload refuses any native addon, and the keyring stand-in exports every export the package declares. - Tests: manifest.test.mjs is new; symbolicate.test.mjs is rewritten on a shared test-only fixture (candidate-fixture.mjs) whose CLI walks up for its project like the real one; the decoder, finalizer, seal and candidate tests gain killing tests. Of 554 mutants over the sweeps and the corrections, 509 are killed, 4 are replaced, and the 41 survivors are equivalent or defensive. - Docs: the contract's 9a.2a section (confinement, native refusal, Git wording, the anchor ruling, a named limit for the unsandboxed capture child), the owned-candidate bullet, the tooling README, the CI and release pack comments, the register checkpoint, two recorded lows (the named limit, and pack-order nondeterminism for Step 9c), the status lines, and two stale Wave 2 links in CLAUDE.md and AGENTS.md. Gates: full pnpm run ci (exit 0, with the real producer and consumer smoke and every causal-control family) and test:acceptance-tools, 881 of 881, both plain and under CI=true GITHUB_ACTIONS=true. Refs: ADR-0111, #46 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 3 +- .github/workflows/release.yml | 3 +- AGENTS.md | 4 +- CLAUDE.md | 4 +- docs/reference/cli/packaging.md | 52 +- docs/roadmap/current.md | 2 +- docs/roadmap/deferred-tasks.md | 28 +- .../phase-2.5.5-hardening-and-remediation.md | 29 +- package.json | 2 +- tools/cli-package/README.md | 12 +- tools/cli-package/candidate-fixture.mjs | 261 ++++ tools/cli-package/candidate.mjs | 48 +- tools/cli-package/candidate.test.mjs | 96 +- tools/cli-package/finalize.test.mjs | 126 ++ tools/cli-package/frame-capture-keyring.mjs | 15 +- tools/cli-package/frame-capture.mjs | 8 +- tools/cli-package/manifest.mjs | 7 +- tools/cli-package/manifest.test.mjs | 507 +++++++ tools/cli-package/seal.test.mjs | 68 +- tools/cli-package/source-map.mjs | 14 +- tools/cli-package/source-map.test.mjs | 50 + tools/cli-package/symbolicate.mjs | 28 +- tools/cli-package/symbolicate.test.mjs | 1180 +++++++++++------ 23 files changed, 2067 insertions(+), 480 deletions(-) create mode 100644 tools/cli-package/candidate-fixture.mjs create mode 100644 tools/cli-package/manifest.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6acae0e9..086c8222 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -637,7 +637,8 @@ jobs: run: pnpm install --frozen-lockfile # The runner builds inlined dependencies fresh in an owned candidate tree, then prepack performs the sole # final CLI build, where original-TS maps are built, validated, staged privately and discarded (Wave 3 - # Step 9a.1); only the tarball and its receipt leave this job. + # Step 9a.1); the seal then writes the private manifest and symbolicates actual packed frames, both + # discarded too (Step 9a.2a); only the tarball and its receipt leave this job. - name: Pack once and independently inspect the public package env: RELAVIUM_PACKAGE_SOURCE_SHA: ${{ github.sha }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 876e307f..cbbb9d92 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -101,7 +101,8 @@ jobs: run: pnpm install --frozen-lockfile # The same required package runner owns fresh dependencies and the sole prepack CLI build in an owned # candidate tree, where original-TS maps are built, validated, staged privately and discarded (Wave 3 - # Step 9a.1); their encrypted release custody is Steps 9b–9d. + # Step 9a.1), with the private manifest and the packed-frame symbolication (Step 9a.2a); their + # encrypted release custody is Steps 9b–9d. - name: Pack once and independently inspect the public package env: RELAVIUM_PACKAGE_SOURCE_SHA: ${{ github.sha }} diff --git a/AGENTS.md b/AGENTS.md index ab8d186c..f2a23147 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,7 +43,7 @@ joins original whole-wave acceptance, maintainer corrections and final reviewed- PR/push CI and SonarCloud success. Published ordinary/enforced coverage records 511 files / 11,026 passing tests / 12 established skips; all four wire and 23 usage mechanisms pass. CR-70/93, CR-95's long continuation and exact-capacity/scoped residuals remain approved deferrals. -The joint [Wave 2 — Shut the doors](docs/roadmap/current.md#next-work--wave-2--shut-the-doors) merged to `main` on 2026-10-15 as PR #94; +The joint [Wave 2 — Shut the doors](docs/roadmap/current.md#wave-2-merged--shut-the-doors) merged to `main` on 2026-10-15 as PR #94; next is [Wave 3 — Clear the ground](docs/roadmap/current.md#wave-3--clear-the-ground). Phase 2.5.5 and Phase 2.6 remain in progress. No W8 implementation or landing gate remains. **Post-closure corrections are independently accepted, 2026-10-10:** core, CLI/DB and tooling/docs @@ -117,7 +117,7 @@ explicit; source CI does not certify later documentation commits. The joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now -accepted on `development`, below; Steps 9a.2–19 have not started) and whole-wave acceptance remains Step 19. +accepted on `development`, below; Step 9a.2a is implemented and under review; Steps 9a.2b–19 have not started) and whole-wave acceptance remains Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification diff --git a/CLAUDE.md b/CLAUDE.md index 030d76ed..13699342 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -130,7 +130,7 @@ throughout the registers describe their original checkpoints. CR-70 (cross-turn tool-history carrying), CR-93 (tenant-scoped mutable state), CR-95's long continuation and existing exact-capacity/scoped residuals retain their approved limits in [deferred-tasks.md](docs/roadmap/deferred-tasks.md). No new deferral, provider capture, billing, -credential or W8 user decision is pending. The joint [Wave 2 — Shut the doors](docs/roadmap/current.md#next-work--wave-2--shut-the-doors) merged to `main` on 2026-10-15 as PR #94; +credential or W8 user decision is pending. The joint [Wave 2 — Shut the doors](docs/roadmap/current.md#wave-2-merged--shut-the-doors) merged to `main` on 2026-10-15 as PR #94; next is [Wave 3 — Clear the ground](docs/roadmap/current.md#wave-3--clear-the-ground). Phase 2.5.5 and Phase 2.6 remain in progress. @@ -206,7 +206,7 @@ explicit; source CI does not certify later documentation commits. The joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now -accepted on `development`, below; Steps 9a.2–19 have not started) and whole-wave acceptance remains Step 19. +accepted on `development`, below; Step 9a.2a is implemented and under review; Steps 9a.2b–19 have not started) and whole-wave acceptance remains Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 1a085f2e..bb19d7f1 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -243,7 +243,12 @@ describes the implemented mechanism; independent acceptance is recorded in the the developer's global excludes file, as the developer's own Git finds it (also through command-scope `GIT_CONFIG_*` configuration) and passed explicitly; that Git reads no home, global or system configuration or attributes and ignores replace refs) into a fresh - owner-only tree, and refuses if any of the three changed during the copy. A link, as a listed file or as any parent directory of one + owner-only tree, and refuses if any of the three changed during the copy. A clean status is + then confirmed byte for byte (Step 9a.2a): each copied file's Git blob id — the hash of its raw + bytes, no filter — must equal the commit tree's entry, because line-ending conversion + (`core.autocrlf`, the Git for Windows default) and Git's stat cache can report converted bytes as + unchanged; a copy that differs makes the identity `dirty`, so the build is symbolicated against + the bytes it was built from and a trusted origin refuses it. A link, as a listed file or as any parent directory of one below the checkout, refuses the copy, so a tracked directory replaced by an ignored link cannot bring in bytes from outside the checkout; so does an untracked nested repository, which Git lists only as its directory. Every build and install pnpm call runs in the @@ -355,31 +360,44 @@ producer. The receipt stays `compiled_package_test` until 9a.2b's product map co - **Private manifest.** After the archive inspection the seal writes `manifest.json` into private staging (owner-only, never overwritten): version 1; the source commit, the commit's Git tree and the `dirty` flag; the package version; the build target; Node, pnpm, TypeScript, tsup and esbuild - versions from the installed tools; SHA-256 digests of the lockfile and every manifest and build - configuration; every final JS file keyed as `dist/.js` with its SHA-256 and its staged map's - path and SHA-256; a SHA-256 for every original source a final map names (repository-relative, no + versions from the installed tools; SHA-256 digests of the lockfile, every package manifest and + every build configuration with the configurations it extends; every final JS file keyed as `dist/.js` with its SHA-256, its map's + package-relative path (`dist/.js.map`) and the staged map's SHA-256; a SHA-256 for every original source a final map names (repository-relative, no content); and the tarball's SHA-256. The staged maps must still be the bytes the finalizer recorded and each JS digest must be the tarball entry's. It holds no host path, credential or source text and is removed with the staging. - **Actual frames.** The seal then runs the finalized `dist` — which the inspection has proven byte-equal to the tarball entry by entry — under a tooling-only `node --import` preload while it - imports a canary workflow (`relavium import`, offline, with its home, project and captured frames - in an owner-only directory of the private root). The preload wraps the external `yaml` package's - `parse`, records V8's structured call sites for the canary document alone, restores the stack - formatter and limit, and replaces the keyring binding with a refusing module, so no native module - loads and no secret is read. Exactly one captured stack is required. + imports a canary workflow (`relavium import`, with its home, project and captured frames in an + owner-only directory of the private root). The project's own `.relavium/` exists before the run, + so the CLI's upward search for a project stops there and never adopts an ancestor's — a + developer's real project on Windows, where the temporary directory lies below the home — and the + run must have imported the canary into it (`CAPTURE_PROJECT`). The preload wraps the external + `yaml` package's `parse`, records V8's structured call sites for the canary document alone, + restores the stack formatter and limit, replaces the keyring package with a module that refuses + every use, and makes any native addon load refuse, so the run loads no native module and reads no + secret. Exactly one captured stack is required. - **Symbolication.** Each frame inside the finalized `dist` must be a file the manifest and the - tarball record (re-hashed at capture), its map must be the staged bytes the manifest records, and - only a frame that lands exactly on a mapping segment counts. Five named call sites must be reached - — three in the CLI and two in the inlined `@relavium/core`, each the one call to a named function - inside another — at the line and UTF-16 column that the matching source revision's syntax tree - gives: the checkout's Git objects at the recorded commit, read by Git without configuration, - filters or replace refs, for a clean candidate, and the candidate copy for a dirty local one, both - bound to the manifest's per-source digests. Wrong JS, a wrong map, permuted map sources and a wrong - source revision therefore cannot reach the anchors. A refusal prints one fixed code + tarball record (re-hashed after the run), its map must be the staged bytes the manifest records, + and only a frame that lands exactly on a mapping segment counts. Five named call sites must be + reached — three in the CLI and two in the inlined `@relavium/core`, each the one call to a named + function inside another, and the table itself refuses fewer than two core or three CLI call sites + — at the line and UTF-16 column that the matching source revision's syntax tree gives: for a + clean candidate the checkout's Git objects at the recorded commit, read as the raw blob (no filter + or text conversion) by Git without home, global or system configuration and without replace refs + (the checkout's own configuration is still read), and for a dirty local one the candidate copy, + both bound to the manifest's per-source digests. Wrong JS, a wrong map, permuted map sources and a + wrong source revision therefore cannot reach the anchors. A refusal prints one fixed code (`PACKED_CLI_MANIFEST_*`, `PACKED_CLI_SYMBOLICATION_*`) and never a frame, path or source text, and no receipt is written. +The capture child is not sandboxed: it runs the candidate CLI's runtime import closure as the +producer's user, with network access, beside the private staging, confined only by its isolated +environment, its owner-only work directory and the refused keyring and native modules. `relavium +import` makes no network request, but nothing enforces that. This is a named limit of the +package-test producer, owned by +[deferred-tasks.md](../../roadmap/deferred-tasks.md#wave-3-step-9a2a--recorded-low-findings-2026-10-24). + ## Recipient custody The maintainer selected encrypted CI archival. GnuPG is release tooling, never a diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index aed40c00..f1454965 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -138,7 +138,7 @@ The [deferred register](deferred-tasks.md) retains its existing limits; closure **Wave 3 Steps 1–8 merged to `main` on 2026-10-18 as [PR #95](https://github.com/HodeTech/Relavium/pull/95), merge commit `6d2a7372` (22:44 UTC). Wave 3 itself is not complete: Steps 9–19 had not started at -that merge (Step 9a.1 is now accepted on `development`; Steps 9a.2–19 have not started) and whole-wave acceptance +that merge (Step 9a.1 is now accepted on `development` and Step 9a.2a is implemented and under review; Steps 9a.2b–19 have not started) and whole-wave acceptance remains Step 19.** The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) joins the completed scope, every review attribution, the independent review's dispositions and the publication state observed at merge. It was the maintainer-authorized, scoped publication checkpoint. diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index 4bf11076..241d1b57 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -2,7 +2,7 @@ > Status: Living -> Last updated: 2026-10-24 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Step 9a.1 is independently accepted on `development` with its recorded low findings below, Steps 9a.2–19 not started; the PR95 increment is closed on `development` under the maintainer's 2026-10-21 round cap). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's edge cases with the maintainer's 2026-10-21 acceptance bar and round cap and the 2026-10-22 rulings. Existing approved deferrals remain. +> Last updated: 2026-10-24 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Step 9a.1 is independently accepted on `development` with its recorded low findings below, Step 9a.2a is implemented and under review with its recorded low findings below, Steps 9a.2b–19 not started; the PR95 increment is closed on `development` under the maintainer's 2026-10-21 round cap). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's edge cases with the maintainer's 2026-10-21 acceptance bar and round cap and the 2026-10-22 rulings. Existing approved deferrals remain. > The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) records completed prerequisites; full M/N lifecycle and other scheduled work remain with their owners. > > Earlier triage: the Phase-2.6 rewrite triaged every open item, the 2026-07-19 full-project @@ -667,6 +667,32 @@ Observed in 9a.1's push CI and its preflight for 9a.2 (2026-10-24): replaces this raw-stack branch (row 10a). **Re-check:** the replacement prints a real error's frames, tested with a thrown `Error` rather than a constructed property. *(low · apps/cli/src/process/render-user-facing.ts)* +## Wave 3 Step 9a.2a — recorded low findings (2026-10-24) + +The internal rigor pass over Step 9a.2a at `1ab4c134`, run before its first review round under the +[2026-10-23 acceptance amendment](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-implementation-checkpoint--2026-10-22), +corrected both high findings, every medium one and every low or nit it could close inside the increment +([checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-implementation-checkpoint--2026-10-24)). +These are recorded instead. **Trigger for every entry:** the next change to the file it names, or a real run +that shows the shape. + +- [ ] **The symbolication child is not sandboxed.** *Low; a named limit.* *(rigor pass, SEC-9A2A-2)* + `captureFrames` runs the candidate CLI's runtime import closure — packages that before 9a.2a ran only in the + consumer jobs — as the producer's user, with network access, in the job whose sibling `staging` directory + holds the private maps and manifest. Its isolated environment, owner-only work directory and the refused + keyring and native addons confine it; nothing enforces that `relavium import` stays offline. packaging.md + names the limit. **Re-check:** when Step 9b designs the custody job, run the capture child under an OS + sandbox or a separate user that cannot read `staging`. *(low · tools/cli-package/symbolicate.mjs)* +- [ ] **Two packs of one source can differ in the packed manifest's key order.** *Low, for Step 9c.* *(rigor + pass, integration lens)* Two productions of the same tree packed the five `@relavium/*` `devDependencies`, + which pnpm rewrites from `workspace:*` to versions at pack time, in different orders (`core, llm, shared, + mcp, db` and `core, db, shared, llm, mcp`; the source order is alphabetical). Every other tarball entry was + byte-equal and the sorted contents agree, but the two tarballs' SHA-256 differ (`84065f84…` and + `d0f5b8da…`). The package-test receipt binds the one tarball it inspected, so nothing is wrong today; a + release candidate simply cannot be rebuilt to the same digest. **Re-check:** in Step 9c's no-rebuild rules, + never compare a rebuilt digest, or make the published manifest's key order deterministic, with a test that + packs twice and compares the digests. *(low · tools/cli-package/prepack.mjs and pnpm's pack step)* + ## Phase 2.6.5 deferrals — `CR-93` and `CR-95` (long half), recorded 2026-09-06; `CR-70`, `CR-94` (remainder) and the in-turn overflow continuation, recorded 2026-09-14; the errored-or-aborted turn residual, the pre-upgrade free-page bytes and the estimator's calibration, recorded 2026-09-18 **Phase closure — 2026-10-11:** W8 merged to `main` in PR #92 and Phase 2.6.5 is complete diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 5ba2a9b8..2e5c98c9 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -9698,6 +9698,29 @@ test oracle, and the receipt stays `compiled_package_test`. On the real product oracle at all 245,806 final-map and 330,759 upstream-map segment starts, every position lies inside its text, and the real producer seals a 25-output, 404-source manifest and reaches all five anchors in about a second. -**Remaining gates.** The internal rigor pass (whole-increment mutation sweep, claims, platform and security -lenses), the full `pnpm run ci`, then one NEW complete cumulative round with no blocker, high or medium finding -and no regression, together with the separate scoped security sitting at the same head and bar. \ No newline at end of file +**Internal rigor pass, 2026-10-24.** Under the maintainer's rigor-before-review rule, a read-only pass covered +the whole increment at `1ab4c134` before its first round: claims, platform, security, integration and critic +lenses and three mutation sweeps (the decoder and finalizer, the manifest, seal and producer, and symbolication and +the preload); the detailed record is local. It reported two high findings. On a clean checkout with converted +line endings (`core.autocrlf`, the Git for Windows default) Git status stays clean while the copy holds CRLF bytes, +so every clean production refused `SOURCE_REVISION`: candidate preparation now confirms a clean status byte for +byte, comparing each copy's Git blob id with the commit tree's entry, and records a differing copy as `dirty`. And +`relavium import --cwd` walks up for a project, so an ancestor `.relavium/` of the temporary directory (a +developer's real project on Windows) could receive the canary: the capture now creates the project's own +`.relavium/` first and refuses `CAPTURE_PROJECT` unless the canary landed there. Medium findings corrected: the +manifest inputs missed the configurations the engine build configurations extend; the two-core, three-CLI anchor +ruling was enforced by nothing; zero-payload VLQ padding decoded as NaN, which passed every comparison, so an eighth +digit now refuses; and many claimed guarantees had no killing test (a new `manifest.test.mjs`, a rewritten +`symbolicate.test.mjs` on a shared test-only fixture whose CLI walks up for its project like the real one, and +extended decoder, finalizer, seal and candidate tests). Lows corrected include a native-addon refusal in the +preload beside the keyring stand-in, which now exports every export the package declares, owners without a body +no longer counted as anchor owners, and the Git, re-hash, README and status wording. Two lows are recorded in +[deferred-tasks.md](../deferred-tasks.md#wave-3-step-9a2a--recorded-low-findings-2026-10-24): the capture child is not +sandboxed (also a named limit in the contract), and two packs of one source can order the packed manifest's +`@relavium/*` dependencies differently, so their digests differ (for Step 9c). All 554 mutants of the three sweeps +and of the corrections were then run against the corrected tests: 509 are killed, 4 targeted code the pass +changed and are replaced by new mutants, and the 41 survivors are equivalent or defensive (unreachable defaults, +caches, a redundant clause, parser options, constants and the binary search's arithmetic). + +**Remaining gates.** The full `pnpm run ci`, then one NEW complete cumulative round with no blocker, high or medium +finding and no regression, together with the separate scoped security sitting at the same head and bar. diff --git a/package.json b/package.json index 8bb79e76..83fded56 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "smoke:budget-replay": "node tools/budget-replay-compat/check.mjs", "check:wire-mutations": "node tools/wire-conformance/check.mjs", "check:usage-mutations": "node tools/usage-coverage-controls/check.mjs", - "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/check.test.mjs tools/cli-package/paths.test.mjs tools/cli-package/source-map.test.mjs tools/cli-package/symbolicate.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", + "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/check.test.mjs tools/cli-package/paths.test.mjs tools/cli-package/source-map.test.mjs tools/cli-package/manifest.test.mjs tools/cli-package/symbolicate.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", "check:fs-mutations": "node tools/fs-policy-controls/check.mjs", "check:secrets-mutations": "node tools/secrets-controls/check.mjs", "smoke:cli-bootstrap": "node tools/cli-bootstrap/check.mjs", diff --git a/tools/cli-package/README.md b/tools/cli-package/README.md index d88fdcd3..a1843f62 100644 --- a/tools/cli-package/README.md +++ b/tools/cli-package/README.md @@ -58,11 +58,13 @@ consumer installs under the temporary directory, so a shared `/tmp` is a named l production prints only prepack's own fixed `✗ CLI …` lines and one fixed code (`production-failure.mjs`), never a build child's captured output, which can quote a map. A corepack-managed pnpm always runs from the developer's own corepack cache, never from the shared -temporary directory that holds the isolated homes (`corepackHome` in `process.mjs`). Before the -receipt is written, the seal decodes the private maps strictly (`source-map.mjs`; the finalizer does -too), seals the private manifest (`manifest.mjs`) and symbolicates actual packed CLI and core -frames, captured by a tooling-only preload during an offline import (`frame-capture.mjs`), against -the matching source revision (`symbolicate.mjs`); all of it is discarded with the staging. The +temporary directory that holds the isolated homes (`corepackHome` in `process.mjs`). The finalizer +decodes every private map strictly (`source-map.mjs`). Before the receipt is written, the seal seals +the private manifest (`manifest.mjs`) and symbolicates actual packed CLI and core frames, captured +by a tooling-only preload while the CLI imports a canary into a private project +(`frame-capture.mjs`), through the strictly decoded maps of the chunks those frames land in against +the matching source revision (`symbolicate.mjs`). The manifest is discarded with the staging, and +the capture's home, project and frames with the private root. The receipt does **not** certify encrypted custody or a production release candidate (Steps 9b–9d), and it stays `compiled_package_test` until the product map controls of Step 9a.2b. No private map, metafile, manifest or per-source inventory enters the public artifact. The release workflow's diff --git a/tools/cli-package/candidate-fixture.mjs b/tools/cli-package/candidate-fixture.mjs new file mode 100644 index 00000000..afa1cb43 --- /dev/null +++ b/tools/cli-package/candidate-fixture.mjs @@ -0,0 +1,261 @@ +/** + * Test-only finalized candidates for the private manifest and symbolication tests. The TypeScript sits + * at the symbolication anchors' real repository paths with the same function names, the CLI imports + * its canary into the project's `.relavium/` found by the same upward search as the real CLI, and the + * exact esbuild behind the installed tsup bundles it with splitting and minify, as the CLI build does. + * The maps are staged with repository-relative sources and recorded as the finalizer records them. + */ +import assert from 'node:assert/strict'; +import { Buffer } from 'node:buffer'; +import { createHash } from 'node:crypto'; +import { + chmodSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + realpathSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import { dirname, join, posix } from 'node:path'; +import { after } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { mapTrailer } from './finalize.mjs'; +import { MANIFEST_INPUTS, sealPrivateManifest } from './manifest.mjs'; + +export const cliRequire = createRequire(new URL('../../apps/cli/package.json', import.meta.url)); +const { build } = createRequire(cliRequire.resolve('tsup/package.json'))('esbuild'); +export const ts = cliRequire('typescript'); +const cliModules = fileURLToPath(new URL('../../apps/cli/node_modules', import.meta.url)); +export const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-candidate-fixture-'))); +after(() => rmSync(scratch, { recursive: true, force: true })); +export const digest = (bytes) => createHash('sha256').update(bytes).digest('hex'); +export const posixModes = process.platform !== 'win32'; +export const PRELOAD = fileURLToPath(new URL('./frame-capture.mjs', import.meta.url)); +let count = 0; +/** A fresh scratch directory name. */ +export const fresh = (name) => join(scratch, `${name}-${(count += 1)}`); + +export const SOURCES = { + 'packages/core/src/yaml-decode.ts': [ + "import { parse as parseYaml } from 'yaml';", + '', + 'export function decodeHardenedYaml(text: string): unknown {', + " return parseYaml(text, { version: '1.2' });", + '}', + '', + ], + 'packages/core/src/parser.ts': [ + "import { decodeHardenedYaml } from './yaml-decode.js';", + '', + 'type Raw = { schema_version?: unknown; workflow?: { id?: unknown } };', + '', + 'export function parseWorkflow(text: string): { id: string } {', + ' const raw = decodeHardenedYaml(text) as Raw;', + " if (raw.schema_version !== '1.0' || typeof raw.workflow?.id !== 'string') {", + " throw new Error('not a workflow');", + ' }', + ' return { id: raw.workflow.id };', + '}', + '', + ], + 'apps/cli/src/authoring/authoring.ts': [ + "import { parseWorkflow } from '../../../../packages/core/src/parser.js';", + '', + 'export function detectAndParse(text: string, fileName: string): { id: string } {', + " if (fileName.endsWith('.relavium.yaml')) {", + ' return parseAsWorkflow(text);', + ' }', + " throw new Error('unsupported');", + '}', + '', + 'function parseAsWorkflow(text: string): { id: string } {', + ' return parseWorkflow(text);', + '}', + '', + ], + 'apps/cli/src/commands/import.ts': [ + "import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';", + "import { basename, dirname, join, resolve } from 'node:path';", + "import { detectAndParse } from '../authoring/authoring.js';", + '', + '// The real CLI walks up from its working directory to the first `.relavium/` (config/paths.ts).', + 'function projectConfigDir(cwd: string): string {', + ' let current = resolve(cwd);', + ' for (;;) {', + " const found = join(current, '.relavium');", + ' if (existsSync(found) && statSync(found).isDirectory()) return found;', + ' const parent = dirname(current);', + " if (parent === current) return join(resolve(cwd), '.relavium');", + ' current = parent;', + ' }', + '}', + '', + 'export function importCommand(path: string, cwd: string): string {', + " const text = readFileSync(path, 'utf8');", + ' const id = detectAndParse(text, basename(path)).id;', + " const target = join(projectConfigDir(cwd), 'workflows', `${id}.relavium.yaml`);", + ' mkdirSync(dirname(target), { recursive: true });', + ' writeFileSync(target, text);', + ' return id;', + '}', + '', + ], + 'apps/cli/src/index.ts': [ + '#!/usr/bin/env node', + "import { Entry } from '@napi-rs/keyring';", + 'try {', + " new Entry('relavium', 'probe');", + '} catch {', + ' // The preload replaces the keyring binding with one that refuses.', + '}', + 'const [command, path, flag, cwd] = process.argv.slice(2);', + "if (command === 'import' && path !== undefined) {", + " const { importCommand } = await import('./commands/import.js');", + " importCommand(path, flag === '--cwd' && cwd !== undefined ? cwd : process.cwd());", + '}', + '', + ], +}; + +/** Write `path` (repository-relative) under `root`. */ +export function put(root, path, bytes) { + mkdirSync(dirname(join(root, ...path.split('/'))), { recursive: true }); + writeFileSync(join(root, ...path.split('/')), bytes); +} + +/** + * A finalized synthetic candidate: its finalized dist, its staged maps with repository-relative + * sources, the finalizer's record, the manifest inputs and the CLI installation, with an owner-only + * staging directory and work directory (under `parent` when given). + */ +export async function candidate({ sources = SOURCES, parent = scratch } = {}) { + const root = fresh('candidate'); + const staging = fresh('staging'); + const work = join(parent, `work-${(count += 1)}`); + for (const directory of [staging, work]) { + mkdirSync(directory, { recursive: true }); + chmodSync(directory, 0o700); + } + mkdirSync(join(staging, 'maps')); + for (const [path, lines] of Object.entries(sources)) put(root, path, lines.join('\n')); + for (const path of MANIFEST_INPUTS) { + put( + root, + path, + path.endsWith('.json') ? `${JSON.stringify({ synthetic: path })}\n` : `# ${path}\n`, + ); + } + put(root, 'apps/cli/tsup.config.ts', "export default {\n target: 'node22',\n};\n"); + put(root, 'apps/cli/package.json', JSON.stringify({ name: 'relavium', type: 'module' })); + symlinkSync(cliModules, join(root, 'apps', 'cli', 'node_modules'), 'junction'); + const cli = join(root, 'apps', 'cli'); + const built = await build({ + absWorkingDir: cli, + entryPoints: { index: 'src/index.ts' }, + outdir: 'dist', + format: 'esm', + platform: 'node', + target: 'node22', + bundle: true, + splitting: true, + minify: true, + sourcemap: 'external', + sourcesContent: false, + external: ['yaml', '@napi-rs/keyring'], + write: false, + logLevel: 'silent', + }); + const outputs = []; + const maps = []; + for (const file of built.outputFiles) { + const name = posix.basename(file.path.split('\\').join('/')); + if (name.endsWith('.map')) { + const map = JSON.parse(file.text); + map.sources = map.sources.map((source) => + posix.normalize(posix.join('apps/cli/dist', source)), + ); + const bytes = Buffer.from(JSON.stringify(map)); + writeFileSync(join(staging, 'maps', name), bytes); + maps.push({ path: `dist/${name}`, bytes: bytes.length, sha256: digest(bytes) }); + } else { + // An external map leaves no link in the code: this is already the finalized JS. + const code = file.text; + assert.equal(code.includes(mapTrailer(name)), false); + put(cli, `dist/${name}`, code); + const bytes = Buffer.from(code); + outputs.push({ path: `dist/${name}`, bytes: bytes.length, sha256: digest(bytes) }); + } + } + writeFileSync( + join(staging, 'finalized.json'), + JSON.stringify({ version: 1, outputs, maps, upstreamMaps: 0 }), + ); + const archive = { + sha256: 'a'.repeat(64), + package: { version: '0.1.1' }, + files: outputs.map((output) => ({ path: `package/${output.path}`, sha256: output.sha256 })), + }; + return { root, cli, staging, work, archive, outputs }; +} + +export const TREE = 'b'.repeat(40); +export const COMMIT = 'c'.repeat(40); +/** Git as the manifest asks it: the commit's tree. */ +export const git = async (command, args) => { + assert.equal(command, 'git'); + assert.deepEqual(args, ['--no-replace-objects', 'rev-parse', '--verify', `${COMMIT}^{tree}`]); + return { stdout: `${TREE}\n`, stderr: '' }; +}; + +/** Seal the manifest of a dirty local candidate, whose source revision is its own copy. */ +export function seal(fixture) { + return sealPrivateManifest({ + candidate: fixture.root, + staging: fixture.staging, + checkout: fixture.root, + source: COMMIT, + dirty: true, + archive: fixture.archive, + run: git, + }); +} + +/** Re-record the finalized digests of the staged maps only, after a deliberate map change. */ +export function rerecordMaps(fixture, mutateRecord = () => {}) { + const record = JSON.parse(readFileSync(join(fixture.staging, 'finalized.json'), 'utf8')); + record.maps = readdirSync(join(fixture.staging, 'maps')).map((name) => { + const bytes = readFileSync(join(fixture.staging, 'maps', name)); + return { path: `dist/${name}`, bytes: bytes.length, sha256: digest(bytes) }; + }); + mutateRecord(record); + writeFileSync(join(fixture.staging, 'finalized.json'), JSON.stringify(record)); +} + +/** Re-record every finalized digest after a deliberate change, then seal a fresh manifest. */ +export function reseal(fixture) { + const outputs = readdirSync(join(fixture.cli, 'dist')).map((name) => { + const bytes = readFileSync(join(fixture.cli, 'dist', name)); + return { path: `dist/${name}`, bytes: bytes.length, sha256: digest(bytes) }; + }); + rerecordMaps(fixture, (record) => (record.outputs = outputs)); + rmSync(join(fixture.staging, 'manifest.json'), { force: true }); + fixture.archive.files = outputs.map((output) => ({ + path: `package/${output.path}`, + sha256: output.sha256, + })); + return seal(fixture); +} + +/** The finalized chunk that holds the core call sites. */ +export function coreChunk(fixture) { + return readdirSync(join(fixture.cli, 'dist')).find((name) => + JSON.parse(readFileSync(join(fixture.staging, 'maps', `${name}.map`), 'utf8')).sources.includes( + 'packages/core/src/parser.ts', + ), + ); +} diff --git a/tools/cli-package/candidate.mjs b/tools/cli-package/candidate.mjs index 8ce20a01..8c32b8f7 100644 --- a/tools/cli-package/candidate.mjs +++ b/tools/cli-package/candidate.mjs @@ -7,6 +7,7 @@ * output, and no build cache is read or written. */ import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; import { chmodSync, existsSync, @@ -315,6 +316,40 @@ export async function dependencyStore({ repository, environment, home, pnpm = ru return store; } +/** + * Whether every copied file is byte for byte the commit's blob. A clean status does not prove it: + * line-ending conversion (`core.autocrlf`, the Git for Windows default) and Git's stat cache can report + * converted working-tree bytes as unchanged, and the candidate is built from those bytes. Each copy's + * Git blob id (SHA-1 of the blob header and its raw bytes, no filter) must equal the commit tree's + * entry for its path. + * @param {{repository: string, candidate: string, source: string, files: readonly string[], + * run?: typeof runCommand}} options + */ +export async function copyMatchesCommit({ + repository, + candidate, + source, + files, + run = runCommand, +}) { + const listing = ( + await run('git', ['--no-replace-objects', 'ls-tree', '-r', '-z', '--full-tree', source], { + cwd: repository, + env: isolatedGitEnvironment(), + }) + ).stdout; + const blobs = new Map(); + for (const entry of gitPaths(listing)) { + const match = /^[0-7]{6} blob ([0-9a-f]{40})\t(.+)$/su.exec(entry); + if (match !== null) blobs.set(match[2], match[1]); + } + return files.every((path) => { + const bytes = readOrdinaryFile(join(candidate, ...path.split('/')), CANDIDATE_LIMITS.fileBytes); + const id = createHash('sha1').update(`blob ${bytes.length}\0`).update(bytes).digest('hex'); + return blobs.get(path) === id; + }); +} + /** * Install the locked closure into the candidate from the store: offline, frozen and without any * lifecycle script, because building or packing the CLI never loads a native module. @@ -405,11 +440,8 @@ export async function prepareCandidate({ pnpm = runPnpm, }) { const identity = await sourceIdentity(repository, run, excludes); - const copied = copyCandidateTree({ - repository, - destination: candidate, - files: await candidateFiles(repository, run, excludes), - }); + const files = await candidateFiles(repository, run, excludes); + const copied = copyCandidateTree({ repository, destination: candidate, files }); const after = await sourceIdentity(repository, run, excludes); assert.ok( after.source === identity.source && @@ -417,8 +449,12 @@ export async function prepareCandidate({ isDeepStrictEqual(after.unchecked, identity.unchecked), 'PACKED_CLI_SOURCE_CHANGED_DURING_COPY', ); + // A clean status is confirmed byte for byte: the candidate is built from the copied bytes. + const dirty = + identity.dirty || + !(await copyMatchesCommit({ repository, candidate, source: identity.source, files, run })); const store = await dependencyStore({ repository, environment, home, pnpm }); await installCandidateDependencies({ candidate, store, env, pnpm }); verifyCandidateLinks(candidate); - return { source: identity.source, dirty: identity.dirty, files: copied.files }; + return { source: identity.source, dirty, files: copied.files }; } diff --git a/tools/cli-package/candidate.test.mjs b/tools/cli-package/candidate.test.mjs index 64de9dfe..31a0f498 100644 --- a/tools/cli-package/candidate.test.mjs +++ b/tools/cli-package/candidate.test.mjs @@ -1,4 +1,6 @@ import assert from 'node:assert/strict'; +import { Buffer } from 'node:buffer'; +import { createHash } from 'node:crypto'; import { chmodSync, closeSync, @@ -24,6 +26,7 @@ import { assertNoBuildCache, candidateFiles, copyCandidateTree, + copyMatchesCommit, dependencyStore, globalExcludesFile, installCandidateDependencies, @@ -806,6 +809,10 @@ test('a candidate build leaves no build cache behind', () => { }); /** A source repository plus fake git and pnpm ports; the fake install creates the workspace links. */ +/** Git's blob object id for raw bytes. */ +const blobId = (bytes) => + createHash('sha1').update(`blob ${bytes.length}\0`).update(bytes).digest('hex'); + function preparation({ status = '', statusAfterCopy = status, @@ -814,6 +821,7 @@ function preparation({ uncheckedAfterCopy = unchecked, excludes, misLink, + convertedPath, } = {}) { const repository = fresh('source'); write(join(repository, 'package.json'), '{}\n'); @@ -833,7 +841,7 @@ function preparation({ let flags = 0; // With an excludes file, the listing and both status calls answer only when they carry it. const prefix = READ(excludes); - const run = async (command, args) => { + const run = async (command, args, options) => { const key = args.join(' '); events.push(`git ${key}`); if (key === '--no-replace-objects rev-parse HEAD') { @@ -851,6 +859,22 @@ function preparation({ flags += 1; return { stdout: `H package.json\0${flags === 1 ? unchecked : uncheckedAfterCopy}` }; } + // The commit's tree: each file's blob id, except `convertedPath`, whose committed blob differs + // from the working-tree bytes (as line-ending conversion leaves it while status stays clean). + if (key === `--no-replace-objects ls-tree -r -z --full-tree ${SHA}`) { + // Asked of the original checkout, through Git without home, global or system configuration. + assert.equal(options.cwd, repository); + assert.deepEqual(options.env, isolatedGitEnvironment()); + return { + stdout: files + .map((path) => { + const bytes = readFileSync(join(repository, ...path.split('/'))); + const committed = path === convertedPath ? Buffer.from('{}\r\n') : bytes; + return `100644 blob ${blobId(committed)}\t${path}`; + }) + .join('\0'), + }; + } throw new Error(`unexpected git ${key}`); }; let installOptions; @@ -912,6 +936,7 @@ test('candidate preparation copies, re-checks identity, installs and verifies li 'git --no-replace-objects rev-parse HEAD', `git ${READ()}status --porcelain=v1 -z --untracked-files=all`, 'git ls-files -v -z', + `git --no-replace-objects ls-tree -r -z --full-tree ${SHA}`, 'pnpm store path', `pnpm install --offline --frozen-lockfile --ignore-scripts --store-dir ${ports.store}`, ]); @@ -922,6 +947,75 @@ test('candidate preparation copies, re-checks identity, installs and verifies li ); }); +test('a copy that differs from the commit is dirty even when status reports the tree clean', async () => { + // Whichever copied file differs. + for (const convertedPath of [ + 'package.json', + ...['shared', 'llm', 'core', 'db', 'mcp'].map((name) => `packages/${name}/package.json`), + ]) { + const ports = preparation({ convertedPath }); + const identity = await prepareCandidate({ + repository: ports.repository, + candidate: ports.candidate, + environment: {}, + home: ports.home, + env: {}, + run: ports.run, + pnpm: ports.pnpm, + }); + assert.deepEqual(identity, { source: SHA, dirty: true, files: 6 }, convertedPath); + } + // An identity that status already calls dirty is not checked again. + const dirty = preparation({ status: ' M package.json\0', statusAfterCopy: ' M package.json\0' }); + await prepareCandidate({ + repository: dirty.repository, + candidate: dirty.candidate, + environment: {}, + home: dirty.home, + env: {}, + run: dirty.run, + pnpm: dirty.pnpm, + }); + assert.equal( + dirty.events.some((event) => event.includes('ls-tree')), + false, + ); +}); + +test('a checkout whose line endings Git converts copies bytes that are not the commit', async () => { + const { repository, inRepository } = await committedRepository('converted-line-endings'); + // core.autocrlf in the checkout's own configuration, which Git reading the checkout honours: + // the file is checked out with CRLF while its blob keeps LF, and status stays clean. + await inRepository(['config', 'core.autocrlf', 'true']); + rmSync(join(repository, 'kept.ts')); + await inRepository(['checkout', '--', 'kept.ts']); + assert.equal(readFileSync(join(repository, 'kept.ts'), 'utf8'), 'export {};\r\n'); + const identity = await sourceIdentity(repository); + assert.equal(identity.dirty, false); + const candidate = fresh('converted-copy'); + copyCandidateTree({ repository, destination: candidate, files: ['kept.ts'] }); + assert.equal( + await copyMatchesCommit({ repository, candidate, source: identity.source, files: ['kept.ts'] }), + false, + ); + // The committed bytes match; a file the commit does not hold does not. + writeFileSync(join(candidate, 'kept.ts'), 'export {};\n'); + assert.equal( + await copyMatchesCommit({ repository, candidate, source: identity.source, files: ['kept.ts'] }), + true, + ); + write(join(candidate, 'extra.ts'), 'export {};\n'); + assert.equal( + await copyMatchesCommit({ + repository, + candidate, + source: identity.source, + files: ['kept.ts', 'extra.ts'], + }), + false, + ); +}); + test('a store the caller names reaches the install, and pnpm is never asked to discover one', async () => { const ports = preparation(); const explicit = fresh('explicit-store'); diff --git a/tools/cli-package/finalize.test.mjs b/tools/cli-package/finalize.test.mjs index c8b58ccf..bd6e71ec 100644 --- a/tools/cli-package/finalize.test.mjs +++ b/tools/cli-package/finalize.test.mjs @@ -779,6 +779,132 @@ test('refuses mappings that are malformed or point outside their texts, upstream } }); +/** Encode absolute segments ([column] or [column, source, line, column, name?]) per generated line. */ +function encode(lines) { + let source = 0; + let line = 0; + let column = 0; + let name = 0; + return lines + .map((segments) => { + let generated = 0; + return segments + .map((segment) => { + let out = vlq(segment[0] - generated); + generated = segment[0]; + if (segment.length >= 4) { + out += vlq(segment[1] - source) + vlq(segment[2] - line) + vlq(segment[3] - column); + [source, line, column] = [segment[1], segment[2], segment[3]]; + } + if (segment.length === 5) { + out += vlq(segment[4] - name); + name = segment[4]; + } + return out; + }) + .join(','); + }) + .join(';'); +} + +test('refuses a name past the names, NaN padding and positions one past their texts', async () => { + const upstreamMap = (candidate) => join(candidate.core, 'dist', 'index.js.map'); + const finalMap = (candidate) => join(candidate.dist, 'index.js.map'); + // A name index one past the names: upstream, which records none, and final. + const unnamed = await fixture(); + editJson(upstreamMap(unnamed), (map) => { + assert.deepEqual(map.names, []); + map.mappings = 'AAAAA'; + }); + assert.equal(refusal(unnamed).code, 'UPSTREAM_MAP_MAPPINGS'); + const named = await fixture(); + let names = 0; + editJson(finalMap(named), (map) => { + names = map.names.length; + map.mappings = `AAAA${vlq(names)}`; + }); + assert.ok(names > 0); + refusesBeforeWriting(named, 'MAP_MAPPINGS'); + // Zero-payload padding that would decode NaN: an upstream name index, and a final generated + // column followed by a segment far past its line, which a NaN column would hide. + const padding = `${'g'.repeat(205)}A`; + const paddedUpstream = await fixture(); + editJson(upstreamMap(paddedUpstream), (map) => (map.mappings = `AAAA${padding}`)); + assert.equal(refusal(paddedUpstream).code, 'UPSTREAM_MAP_MAPPINGS'); + const paddedFinal = await fixture(); + editJson(finalMap(paddedFinal), (map) => (map.mappings = `${padding}AAA,${vlq(100000)}`)); + refusesBeforeWriting(paddedFinal, 'MAP_MAPPINGS'); + // The final text is measured without the stripped trailer: the finalized index.js ends with a + // newline, so column 1 of its empty last line is past it. + const trailer = await fixture(); + editJson(finalMap(trailer), (map) => { + const code = trailer.emitted['index.js']; + assert.ok(code.endsWith('\n')); + map.mappings += `${';'.repeat(code.split('\n').length - map.mappings.split(';').length)}C`; + }); + refusesBeforeWriting(trailer, 'MAP_RANGE'); + // A segment one line past the generated text, final and upstream. + const pastFinal = await fixture(); + editJson(finalMap(pastFinal), (map) => { + const lines = pastFinal.emitted['index.js'].split('\n').length; + map.mappings += `${';'.repeat(lines - map.mappings.split(';').length + 1)}A`; + }); + refusesBeforeWriting(pastFinal, 'MAP_RANGE'); + const pastUpstream = await fixture(); + const js = readFileSync(join(pastUpstream.core, 'dist', 'index.js'), 'utf8'); + editJson(upstreamMap(pastUpstream), (map) => { + map.mappings += `${';'.repeat(js.split('\n').length - map.mappings.split(';').length + 1)}A`; + }); + assert.equal(refusal(pastUpstream).code, 'UPSTREAM_MAP_RANGE'); + // An original line one past its source, and a column one past its line's end. + const source = readFileSync(join(pastUpstream.core, 'src', 'index.ts'), 'utf8').split('\n'); + for (const mappings of [`AA${vlq(source.length)}A`, `AAA${vlq(source[0].length + 1)}`]) { + const candidate = await fixture(); + editJson(upstreamMap(candidate), (map) => (map.mappings = mappings)); + assert.equal(refusal(candidate).code, 'UPSTREAM_MAP_RANGE', mappings); + } +}); + +test('checks each original position against its own source, upstream and final', async () => { + // The .tsx fixture bundles index.ts and the one-line view.tsx into index.js: two sources. + const invalid = await fixture({ tsx: true }); + let view = -1; + editJson(join(invalid.dist, 'index.js.map'), (map) => { + assert.equal(map.sources.length, 2); + view = map.sources.findIndex((source) => source.endsWith('view.tsx')); + // Line 5 exists in index.ts but not in view.tsx. + map.mappings = encode([[], [[0, view, 5, 0]]]); + }); + assert.ok(view >= 0); + refusesBeforeWriting(invalid, 'MAP_RANGE'); + // The last line of index.ts and the caret end of view.tsx's only line are inside their own source. + const valid = await fixture({ tsx: true }); + editJson(join(valid.dist, 'index.js.map'), (map) => { + const index = map.sources.findIndex((source) => source.endsWith('index.ts')); + const indexLines = readFileSync(join(valid.cli, 'src', 'index.ts'), 'utf8').split('\n'); + const viewLines = readFileSync(join(valid.cli, 'src', 'view.tsx'), 'utf8').split('\n'); + assert.ok(indexLines.length > viewLines.length + 2); + map.mappings = encode([ + [], + [ + [0, index, indexLines.length - 2, indexLines[indexLines.length - 2].length], + [1, 1 - index, 0, viewLines[0].length], + ], + ]); + }); + finalizeCandidate({ repository: valid.root, staging: valid.staging }); + // An upstream map with two sources: column 25 of the first line is inside index.ts, not types.ts. + const upstream = await fixture(); + const types = readFileSync(join(upstream.core, 'src', 'types.ts'), 'utf8').split('\n'); + const index = readFileSync(join(upstream.core, 'src', 'index.ts'), 'utf8').split('\n'); + assert.ok(types[0].length < 25 && index[0].length >= 25); + editJson(join(upstream.core, 'dist', 'index.js.map'), (map) => { + map.sources = ['../src/index.ts', '../src/types.ts']; + map.mappings = encode([[[0, 1, 0, 25]]]); + }); + assert.equal(refusal(upstream).code, 'UPSTREAM_MAP_RANGE'); +}); + test('admits original .tsx CLI sources', async () => { const candidate = await fixture({ tsx: true }); finalizeCandidate({ repository: candidate.root, staging: candidate.staging }); diff --git a/tools/cli-package/frame-capture-keyring.mjs b/tools/cli-package/frame-capture-keyring.mjs index 21daabda..7c4a38ca 100644 --- a/tools/cli-package/frame-capture-keyring.mjs +++ b/tools/cli-package/frame-capture-keyring.mjs @@ -1,6 +1,17 @@ -// The keyring binding seen by the symbolication run (frame-capture-hooks.mjs): every use refuses. +// The keyring binding seen by the symbolication run (frame-capture-hooks.mjs): every export of the +// real package is present, and every use refuses, so no native binding loads and no secret is read. +const refused = () => { + throw new Error('The symbolication run refuses the keyring'); +}; export class Entry { constructor() { - throw new Error('The symbolication run refuses the keyring'); + refused(); } } +export class AsyncEntry extends Entry {} +export class EntryTask extends Entry {} +export class SecretTask extends Entry {} +export class PasswordTask extends Entry {} +export class FindCredentials extends Entry {} +export const findCredentials = refused; +export const findCredentialsAsync = refused; diff --git a/tools/cli-package/frame-capture.mjs b/tools/cli-package/frame-capture.mjs index 58d92a84..7ff0ca95 100644 --- a/tools/cli-package/frame-capture.mjs +++ b/tools/cli-package/frame-capture.mjs @@ -5,7 +5,8 @@ * instance the CLI's import and require both reach — and records V8's structured call sites for the * canary document alone, so one synchronous stack crosses the CLI and the inlined `@relavium/core`. It * writes one owner-only record when the process exits; the producer refuses unless exactly one stack - * was captured. The keyring binding is replaced by a refusing module (frame-capture-hooks.mjs). + * was captured. The keyring binding is replaced by a refusing module (frame-capture-hooks.mjs) and + * any native module load refuses. */ import { closeSync, openSync, writeSync } from 'node:fs'; import { createRequire, register } from 'node:module'; @@ -16,6 +17,11 @@ const canary = process.env.RELAVIUM_FRAME_CAPTURE_CANARY; if (!log || !manifest || !canary) throw new Error('The symbolication preload is not configured'); register(new URL('./frame-capture-hooks.mjs', import.meta.url)); +// The import route needs no native module; any addon load refuses instead of running native code +// beside the private staging. +process.dlopen = () => { + throw new Error('The symbolication run refuses native modules'); +}; /** @type {{file: string | null, line: number | null, column: number | null, async: boolean}[][]} */ const captures = []; diff --git a/tools/cli-package/manifest.mjs b/tools/cli-package/manifest.mjs index 88e85f50..78237d63 100644 --- a/tools/cli-package/manifest.mjs +++ b/tools/cli-package/manifest.mjs @@ -16,7 +16,10 @@ import { isolatedGitEnvironment, runCommand } from './process.mjs'; export const MANIFEST_FILE = 'manifest.json'; -/** The candidate files whose bytes decide the build: the lock and every manifest and build configuration. */ +/** + * The candidate files whose bytes decide the build: the lock, every package manifest, and every build + * configuration with the configurations it extends. + */ export const MANIFEST_INPUTS = Object.freeze([ '.npmrc', 'package.json', @@ -28,8 +31,10 @@ export const MANIFEST_INPUTS = Object.freeze([ 'apps/cli/tsup.config.ts', ...ENGINE_PACKAGES.flatMap((name) => [ `packages/${name}/package.json`, + `packages/${name}/tsconfig.json`, `packages/${name}/tsconfig.build.json`, ]), + 'packages/core/tsconfig.purity.json', ]); const SHA256 = /^[0-9a-f]{64}$/u; diff --git a/tools/cli-package/manifest.test.mjs b/tools/cli-package/manifest.test.mjs new file mode 100644 index 00000000..f4b455ca --- /dev/null +++ b/tools/cli-package/manifest.test.mjs @@ -0,0 +1,507 @@ +import assert from 'node:assert/strict'; +import { Buffer } from 'node:buffer'; +import { createRequire } from 'node:module'; +import { existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'; +import { join, posix } from 'node:path'; +import { test } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { + COMMIT, + SOURCES, + TREE, + candidate, + cliRequire, + coreChunk, + digest, + fresh, + git, + posixModes, + rerecordMaps, + scratch, + seal, + ts, +} from './candidate-fixture.mjs'; +import { ENGINE_PACKAGES } from './finalize.mjs'; +import { + MANIFEST_INPUTS, + ManifestError, + readPrivateManifest, + sealPrivateManifest, +} from './manifest.mjs'; + +const refuses = (code) => (error) => error instanceof ManifestError && error.code === code; +const repository = fileURLToPath(new URL('../../', import.meta.url)); + +test('the manifest inputs are every build configuration the real repository extends', () => { + // Each configuration a build reads, followed through its `extends` chain, is a digested input. + const roots = [ + 'apps/cli/tsconfig.json', + ...ENGINE_PACKAGES.map((name) => `packages/${name}/tsconfig.build.json`), + ]; + const reached = new Set(); + const follow = (path) => { + reached.add(path); + const { config, error } = ts.readConfigFile( + join(repository, ...path.split('/')), + ts.sys.readFile, + ); + assert.equal(error, undefined, path); + if (config.extends === undefined) return; + assert.equal(typeof config.extends, 'string', path); + assert.match(config.extends, /^\.\.?\//u, `${path} extends a relative configuration`); + follow(posix.normalize(posix.join(posix.dirname(path), config.extends))); + }; + for (const root of roots) follow(root); + // With the lock, the workspace and npm configuration, the tsup configuration and every package + // manifest the build installs, that is exactly the input list. + const expected = new Set([ + ...reached, + '.npmrc', + 'package.json', + 'pnpm-lock.yaml', + 'pnpm-workspace.yaml', + 'apps/cli/package.json', + 'apps/cli/tsup.config.ts', + ...ENGINE_PACKAGES.map((name) => `packages/${name}/package.json`), + ]); + assert.equal(MANIFEST_INPUTS.length, expected.size); + assert.deepEqual(new Set(MANIFEST_INPUTS), expected); + for (const path of MANIFEST_INPUTS) + assert.ok(existsSync(join(repository, ...path.split('/'))), path); +}); + +test('the manifest binds the commit, toolchain, inputs, finalized pairs, sources and tarball', async () => { + const fixture = await candidate(); + const manifest = await seal(fixture); + assert.deepEqual(manifest.source, { commit: COMMIT, tree: TREE, dirty: true }); + assert.equal(manifest.packageVersion, '0.1.1'); + assert.equal(manifest.target, 'node22'); + assert.equal(manifest.tarballSha256, 'a'.repeat(64)); + assert.equal(manifest.toolchain.node, process.version); + assert.equal(manifest.toolchain.pnpm, '9.12.3'); + assert.equal(manifest.toolchain.typescript, ts.version); + // The installed tsup, and the esbuild tsup itself resolves. + assert.equal(manifest.toolchain.tsup, cliRequire('tsup/package.json').version); + assert.equal( + manifest.toolchain.esbuild, + createRequire(cliRequire.resolve('tsup/package.json'))('esbuild/package.json').version, + ); + assert.deepEqual(Object.keys(manifest.inputs), [...MANIFEST_INPUTS]); + for (const path of MANIFEST_INPUTS) { + assert.equal( + manifest.inputs[path], + digest(readFileSync(join(fixture.root, ...path.split('/')))), + path, + ); + } + assert.deepEqual( + manifest.outputs.map((output) => output.path), + fixture.outputs.map((output) => output.path).sort(), + ); + for (const output of manifest.outputs) { + assert.equal(output.sha256, digest(readFileSync(join(fixture.cli, output.path)))); + assert.deepEqual(output.map, { + path: `${output.path}.map`, + sha256: digest( + readFileSync(join(fixture.staging, 'maps', `${posix.basename(output.path)}.map`)), + ), + }); + } + assert.deepEqual(Object.keys(manifest.sources), Object.keys(SOURCES).sort()); + for (const [path, value] of Object.entries(manifest.sources)) { + assert.equal(value, digest(readFileSync(join(fixture.root, ...path.split('/'))))); + } + // Written once, owner-only, holding no absolute path, and read back exactly. + const text = readFileSync(join(fixture.staging, 'manifest.json'), 'utf8'); + assert.deepEqual(JSON.parse(text), manifest); + assert.deepEqual(readPrivateManifest(fixture.staging), manifest); + assert.equal(text.includes(scratch), false); + if (posixModes) + assert.equal(statSync(join(fixture.staging, 'manifest.json')).mode & 0o777, 0o600); + await assert.rejects(seal(fixture), (error) => error.code === 'EEXIST'); +}); + +test('the package version and tarball digest are the inspected archive’s', async () => { + const fixture = await candidate(); + fixture.archive.package.version = '9.8.7'; + fixture.archive.sha256 = 'e'.repeat(64); + const manifest = await seal(fixture); + assert.equal(manifest.packageVersion, '9.8.7'); + assert.equal(manifest.tarballSha256, 'e'.repeat(64)); +}); + +test('the tree is asked of the original checkout through isolated Git', async () => { + const fixture = await candidate(); + // Sources and toolchain come from the candidate, so an absent checkout serves only Git. + const checkout = join(scratch, 'absent-checkout'); + let seen; + await sealPrivateManifest({ + candidate: fixture.root, + staging: fixture.staging, + checkout, + source: COMMIT, + dirty: false, + archive: fixture.archive, + run: async (command, args, options) => { + seen = options; + return git(command, args); + }, + }); + assert.equal(seen.cwd, checkout); + assert.equal(seen.env.GIT_CONFIG_NOSYSTEM, '1'); + assert.equal(seen.env.GIT_NO_REPLACE_OBJECTS, '1'); + assert.equal(seen.env.GIT_DIR, undefined); + assert.ok(Number.isFinite(seen.timeoutMs) && seen.timeoutMs >= 10_000); +}); + +test('the manifest refuses drifted maps and tarball entries, unpaired records and a missing tree', async () => { + for (const [label, change, code] of [ + [ + 'a staged map changed after the finalizer recorded it', + (fixture) => + writeFileSync(join(fixture.staging, 'maps', `${coreChunk(fixture)}.map`), '{"sources":[]}'), + 'MAP_DRIFT', + ], + [ + 'a staged map changed to bytes of the same length', + (fixture) => { + const path = join(fixture.staging, 'maps', `${coreChunk(fixture)}.map`); + const text = readFileSync(path, 'utf8'); + assert.equal(text.startsWith('{"version":3,'), true); + writeFileSync(path, text.replace('{"version":3,', '{"version":4,')); + }, + 'MAP_DRIFT', + ], + [ + 'a recorded map size that disagrees with its bytes', + (fixture) => { + const name = coreChunk(fixture); + rerecordMaps(fixture, (record) => { + record.maps.find((map) => map.path === `dist/${name}.map`).bytes += 1; + }); + }, + 'MAP_DRIFT', + ], + [ + 'a recorded map that is not JSON', + (fixture) => { + writeFileSync(join(fixture.staging, 'maps', `${coreChunk(fixture)}.map`), 'not json'); + rerecordMaps(fixture); + }, + 'MAP_DRIFT', + ], + [ + 'a recorded map without sources', + (fixture) => { + writeFileSync(join(fixture.staging, 'maps', `${coreChunk(fixture)}.map`), '{"version":3}'); + rerecordMaps(fixture); + }, + 'MAP_DRIFT', + ], + [ + 'a tarball entry that is not the finalized JS', + (fixture) => (fixture.archive.files[0].sha256 = 'd'.repeat(64)), + 'TARBALL_DRIFT', + ], + [ + 'an output without its map in the record', + (fixture) => rerecordMaps(fixture, (record) => record.maps.pop()), + 'RECORD', + ], + [ + 'a map that pairs no output', + (fixture) => + rerecordMaps(fixture, (record) => + record.maps.push({ path: 'dist/unpaired.js.map', bytes: 2, sha256: digest('{}') }), + ), + 'RECORD', + ], + [ + 'a null map entry', + (fixture) => rerecordMaps(fixture, (record) => record.maps.push(null)), + 'RECORD', + ], + [ + 'an output outside dist', + (fixture) => { + const name = coreChunk(fixture); + rerecordMaps(fixture, (record) => { + record.outputs.find((output) => output.path === `dist/${name}`).path = `dist/../${name}`; + record.maps.find((map) => map.path === `dist/${name}.map`).path = `dist/../${name}.map`; + }); + }, + 'RECORD', + ], + [ + 'outputs that are not a list', + (fixture) => + writeFileSync( + join(fixture.staging, 'finalized.json'), + JSON.stringify({ outputs: {}, maps: [] }), + ), + 'RECORD', + ], + ['a missing record', (fixture) => rmSync(join(fixture.staging, 'finalized.json')), 'RECORD'], + [ + 'a build configuration without its target', + (fixture) => writeFileSync(join(fixture.cli, 'tsup.config.ts'), 'export default {};\n'), + 'TARGET', + ], + [ + 'a build configuration with two top-level targets', + (fixture) => + writeFileSync( + join(fixture.cli, 'tsup.config.ts'), + "export default {\n target: 'node20',\n target: 'node22',\n};\n", + ), + 'TARGET', + ], + [ + 'a target that is not a Node target (the seal validates what it writes)', + (fixture) => + writeFileSync( + join(fixture.cli, 'tsup.config.ts'), + "export default {\n target: 'es2022',\n};\n", + ), + 'SHAPE', + ], + [ + 'a missing manifest input', + (fixture) => rmSync(join(fixture.root, 'pnpm-lock.yaml')), + 'UNREADABLE_INPUT', + ], + [ + 'an uninstalled toolchain', + (fixture) => rmSync(join(fixture.root, 'apps', 'cli', 'node_modules')), + 'TOOLCHAIN', + ], + ]) { + const fixture = await candidate(); + change(fixture); + await assert.rejects(seal(fixture), refuses(code), label); + assert.equal(existsSync(join(fixture.staging, 'manifest.json')), false, label); + } + const fixture = await candidate(); + for (const run of [ + async () => ({ stdout: 'not a tree\n', stderr: '' }), + async () => { + throw new Error('git failed'); + }, + ]) { + await assert.rejects( + sealPrivateManifest({ + candidate: fixture.root, + staging: fixture.staging, + checkout: fixture.root, + source: COMMIT, + dirty: false, + archive: fixture.archive, + run, + }), + refuses('SOURCE_TREE'), + ); + } +}); + +test('only the top-level target line of the build configuration counts', async () => { + const fixture = await candidate(); + writeFileSync( + join(fixture.cli, 'tsup.config.ts'), + "// target: 'node18',\nexport default {\n // target: 'node20',\n esbuildOptions: {\n target: 'node20',\n },\n target: 'node22',\n};\n", + ); + assert.equal((await seal(fixture)).target, 'node22'); +}); + +test('a final map naming a source outside the admitted roots refuses before it is read', async () => { + writeFileSync(join(scratch, 'outside.ts'), 'export {};\n'); + for (const name of ['apps/cli/src/../../../../outside.ts', 'node_modules/yaml/dist/index.ts']) { + const fixture = await candidate(); + const path = join(fixture.staging, 'maps', `${coreChunk(fixture)}.map`); + const map = JSON.parse(readFileSync(path, 'utf8')); + map.sources.push(name); + writeFileSync(path, JSON.stringify(map)); + rerecordMaps(fixture); + await assert.rejects(seal(fixture), refuses('SOURCE_PATH'), name); + assert.equal(existsSync(join(fixture.staging, 'manifest.json')), false, name); + } +}); + +test('every manifest read is bounded', async () => { + const pad = (text, size) => text + ' '.repeat(Math.max(0, size - Buffer.byteLength(text))); + for (const [label, change, code] of [ + [ + 'build configuration', + (fixture) => { + const path = join(fixture.cli, 'tsup.config.ts'); + writeFileSync(path, pad(readFileSync(path, 'utf8'), 128 * 1024 + 1)); + }, + 'UNREADABLE_INPUT', + ], + [ + 'lockfile', + (fixture) => + writeFileSync(join(fixture.root, 'pnpm-lock.yaml'), pad('# lock\n', 16 * 1024 * 1024 + 1)), + 'UNREADABLE_INPUT', + ], + [ + 'source', + (fixture) => { + const path = join(fixture.root, 'packages', 'core', 'src', 'parser.ts'); + writeFileSync(path, pad(readFileSync(path, 'utf8'), 8 * 1024 * 1024 + 1)); + }, + 'UNREADABLE_INPUT', + ], + [ + 'map', + (fixture) => { + const path = join(fixture.staging, 'maps', `${coreChunk(fixture)}.map`); + writeFileSync(path, pad(readFileSync(path, 'utf8'), 16 * 1024 * 1024 + 1)); + rerecordMaps(fixture); + }, + 'UNREADABLE_INPUT', + ], + [ + 'record', + (fixture) => { + const path = join(fixture.staging, 'finalized.json'); + writeFileSync(path, pad(readFileSync(path, 'utf8'), 1024 * 1024 + 1)); + }, + 'RECORD', + ], + ]) { + const fixture = await candidate(); + change(fixture); + await assert.rejects(seal(fixture), refuses(code), label); + } + const fixture = await candidate(); + await seal(fixture); + const path = join(fixture.staging, 'manifest.json'); + const text = readFileSync(path, 'utf8'); + rmSync(path); + writeFileSync(path, pad(text, 4 * 1024 * 1024 + 1)); + assert.throws(() => readPrivateManifest(fixture.staging), refuses('UNREADABLE')); + const empty = fresh('no-manifest'); + mkdirSync(empty); + assert.throws(() => readPrivateManifest(empty), refuses('UNREADABLE')); +}); + +test('a refusal is a ManifestError naming only its fixed code', () => { + const empty = fresh('no-manifest'); + mkdirSync(empty); + assert.throws(() => readPrivateManifest(empty), { + name: 'ManifestError', + code: 'UNREADABLE', + message: 'PACKED_CLI_MANIFEST_UNREADABLE', + }); +}); + +test('a sealed manifest that is not exactly the validated shape refuses on read', async () => { + const fixture = await candidate(); + const valid = await seal(fixture); + const hex = (c, n = 64) => c.repeat(n); + const many = (n, make) => Array.from({ length: n }, (_, i) => make(String(i).padStart(5, '0'))); + const output = (name) => ({ + path: `dist/${name}.js`, + sha256: hex('1'), + map: { path: `dist/${name}.js.map`, sha256: hex('2') }, + }); + const cases = [ + ['extra top-level key', (m) => (m.extra = 1)], + ['missing top-level key', (m) => delete m.target], + ['version 2', (m) => (m.version = 2)], + ['source extra key', (m) => (m.source.extra = 1)], + ['commit not hex', (m) => (m.source.commit = 'x')], + ['commit with suffix', (m) => (m.source.commit = `${hex('c', 40)}0`)], + ['commit short', (m) => (m.source.commit = hex('c', 39))], + ['tree not hex', (m) => (m.source.tree = 'x')], + ['dirty as string', (m) => (m.source.dirty = 'true')], + ['packageVersion as array', (m) => (m.packageVersion = ['0.1.1'])], + ['packageVersion not semver', (m) => (m.packageVersion = '0.1')], + ['packageVersion with trailing text', (m) => (m.packageVersion = '0.1.1 x')], + ['packageVersion oversized suffix', (m) => (m.packageVersion = `0.1.1-${'a'.repeat(65)}`)], + ['target as array', (m) => (m.target = ['node22'])], + ['target not nodeNN', (m) => (m.target = 'es2022')], + ['toolchain extra key', (m) => (m.toolchain.extra = '1.0.0')], + ['node version malformed', (m) => (m.toolchain.node = 'v22')], + ['pnpm other', (m) => (m.toolchain.pnpm = '9.12.4')], + ['typescript malformed', (m) => (m.toolchain.typescript = 'x')], + ['tsup malformed', (m) => (m.toolchain.tsup = 'x')], + ['esbuild malformed', (m) => (m.toolchain.esbuild = 'x')], + ['inputs missing one', (m) => delete m.inputs['.npmrc']], + ['inputs extra one', (m) => (m.inputs['extra.json'] = hex('3'))], + ['input digest not hex', (m) => (m.inputs['.npmrc'] = 'x')], + ['input digest uppercase', (m) => (m.inputs['.npmrc'] = hex('A'))], + ['input digest short', (m) => (m.inputs['.npmrc'] = hex('3', 63))], + ['input digest with prefix', (m) => (m.inputs['.npmrc'] = `z${hex('3')}`)], + ['tarball digest not hex', (m) => (m.tarballSha256 = 'x')], + ['tarball digest as array', (m) => (m.tarballSha256 = [hex('a')])], + ['outputs not an array', (m) => (m.outputs = {})], + ['outputs empty', (m) => (m.outputs = [])], + ['outputs over the bound', (m) => (m.outputs = many(257, (i) => output(`c${i}`)))], + ['output extra key', (m) => (m.outputs[0].extra = 1)], + // Paired map paths and the order stay valid, so only the path pattern can refuse these. + [ + 'output path escaping', + (m) => { + m.outputs[0].path = 'dist/../x.js'; + m.outputs[0].map.path = 'dist/../x.js.map'; + }, + ], + [ + 'output path not anchored', + (m) => { + m.outputs.at(-1).path = 'x/dist/zz.js'; + m.outputs.at(-1).map.path = 'x/dist/zz.js.map'; + }, + ], + ['output digest not hex', (m) => (m.outputs[0].sha256 = 'x')], + ['output map extra key', (m) => (m.outputs[0].map.extra = 1)], + ['output map path not paired', (m) => (m.outputs[0].map.path = 'dist/other.js.map')], + ['output map digest not hex', (m) => (m.outputs[0].map.sha256 = 'x')], + ['outputs unsorted', (m) => m.outputs.reverse()], + ['outputs duplicated', (m) => m.outputs.push(m.outputs.at(-1))], + ['sources an array', (m) => (m.sources = [])], + ['sources null', (m) => (m.sources = null)], + ['sources empty', (m) => (m.sources = {})], + [ + 'sources over the bound', + (m) => + (m.sources = Object.fromEntries(many(8193, (i) => [`apps/cli/src/s${i}.ts`, hex('4')]))), + ], + [ + 'source outside the admitted roots', + (m) => (m.sources['node_modules/yaml/index.ts'] = hex('4')), + ], + [ + 'source escaping by dot segments', + (m) => (m.sources['apps/cli/src/../../../x.ts'] = hex('4')), + ], + ['source not anchored', (m) => (m.sources['x/apps/cli/src/a.ts'] = hex('4'))], + ['source in another package', (m) => (m.sources['packages/other/src/a.ts'] = hex('4'))], + ['source outside src', (m) => (m.sources['apps/cli/a.ts'] = hex('4'))], + ['source not TypeScript', (m) => (m.sources['apps/cli/src/a.js'] = hex('4'))], + ['source digest not hex', (m) => (m.sources[Object.keys(m.sources)[0]] = 'x')], + [ + 'sources unsorted', + (m) => (m.sources = Object.fromEntries(Object.entries(m.sources).reverse())), + ], + ]; + for (const [label, change] of cases) { + const variant = JSON.parse(JSON.stringify(valid)); + change(variant); + // Keep sources sorted after an addition unless the case is about order. + if ( + label !== 'sources unsorted' && + variant.sources !== null && + !Array.isArray(variant.sources) + ) { + variant.sources = Object.fromEntries( + Object.entries(variant.sources).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)), + ); + } + const staging = fresh('shape'); + mkdirSync(staging); + writeFileSync(join(staging, 'manifest.json'), JSON.stringify(variant)); + assert.throws(() => readPrivateManifest(staging), refuses('SHAPE'), label); + } +}); diff --git a/tools/cli-package/seal.test.mjs b/tools/cli-package/seal.test.mjs index 286deb9f..bd527d44 100644 --- a/tools/cli-package/seal.test.mjs +++ b/tools/cli-package/seal.test.mjs @@ -16,6 +16,7 @@ import { dirname, join } from 'node:path'; import { test } from 'node:test'; import { fileURLToPath } from 'node:url'; import { gzipSync } from 'node:zlib'; +import { ManifestError } from './manifest.mjs'; import { finalizedPublicFiles, producerNodeMajor, sealPackage } from './seal.mjs'; import { ustar } from './ustar-fixture.mjs'; @@ -368,7 +369,7 @@ const receiptPath = (artifact) => join(artifact, 'package-test-receipt.json'); /** * The seal with recording stand-ins for the private manifest and symbolication, whose own tests - * (symbolicate.test.mjs) run them for real. + * (manifest.test.mjs, symbolicate.test.mjs) run them for real. */ function sealWith(artifact, candidate, calls = []) { return sealPackage(artifact, { @@ -554,6 +555,71 @@ test('a refused manifest or symbolication leaves no receipt, and an unproven arc }); }); +test('both steps receive the candidate, the original checkout and the dirty flag as given', async () => { + await withCandidate(async ({ root, staging, artifact }) => { + const calls = []; + const checkout = join(dirname(staging), 'original-checkout'); + await withOrigin({}, () => + sealPackage(artifact, { + repository: root, + staging, + source: SOURCE, + dirty: true, + checkout, + work: join(dirname(staging), 'work'), + ports: { + manifest: async (options) => void calls.push(options), + symbolicate: async (options) => void calls.push(options), + }, + }), + ); + const [manifest, symbolicate] = calls; + assert.deepEqual( + [manifest.candidate, manifest.checkout, manifest.dirty], + [root, checkout, true], + ); + assert.deepEqual([symbolicate.candidate, symbolicate.checkout], [root, checkout]); + }); +}); + +test('without ports the seal runs the real private manifest and the real symbolication', async () => { + // The real manifest refuses this candidate's record, which stages no maps. + await withCandidate(async ({ root, staging, artifact }) => { + await withOrigin({}, () => + assert.rejects( + sealPackage(artifact, { + repository: root, + staging, + source: SOURCE, + dirty: false, + checkout: root, + work: join(dirname(staging), 'work'), + }), + (error) => error instanceof ManifestError && error.code === 'RECORD', + ), + ); + assert.equal(existsSync(receiptPath(artifact)), false); + }); + // The real symbolication reads the manifest that a stand-in never sealed. + await withCandidate(async ({ root, staging, artifact }) => { + await withOrigin({}, () => + assert.rejects( + sealPackage(artifact, { + repository: root, + staging, + source: SOURCE, + dirty: false, + checkout: root, + work: join(dirname(staging), 'work'), + ports: { manifest: async () => {} }, + }), + (error) => error instanceof ManifestError && error.code === 'UNREADABLE', + ), + ); + assert.equal(existsSync(receiptPath(artifact)), false); + }); +}); + test('the seal requires the checkout and a private work directory before it reads the candidate', async () => { await withCandidate(async ({ staging, artifact }) => { const missing = join(artifact, 'no-candidate'); diff --git a/tools/cli-package/source-map.mjs b/tools/cli-package/source-map.mjs index d7df73ad..a27e35bd 100644 --- a/tools/cli-package/source-map.mjs +++ b/tools/cli-package/source-map.mjs @@ -24,6 +24,8 @@ const DIGITS = new Int8Array(128).fill(-1); for (let index = 0; index < BASE64.length; index += 1) DIGITS[BASE64.charCodeAt(index)] = index; /** The largest unsigned VLQ value (a signed 32-bit magnitude and its sign bit). */ const VLQ_MAX = 0xffffffff; +/** The most base64 digits a value up to `VLQ_MAX` can take (five bits each). */ +const MAX_DIGITS = 7; /** * One decoded segment: a generated column, and for a mapped segment its source index and 0-based @@ -34,9 +36,10 @@ const VLQ_MAX = 0xffffffff; /** * Decode `mappings` into one array of segments per generated line. Refuses a character outside the - * base64 alphabet, a truncated or oversized value, an empty segment, a segment of other than 1, 4 or 5 - * fields, a generated column that does not increase along its line, and a negative or out-of-range - * source, name, line or column. + * base64 alphabet, a truncated, oversized or over-long value, an empty segment, a segment of other + * than 1, 4 or 5 fields, a generated column that does not increase along its line, a negative line or + * column, and an out-of-range source or name index. Upper bounds on positions are the range checks' + * (`assertGeneratedRange`, `assertOriginalRange`). * @param {string} mappings * @param {{sources: number, names: number}} counts the lengths of the map's `sources` and `names` * @returns {Segment[][]} @@ -99,7 +102,7 @@ export function decodeMappings(mappings, { sources, names }) { } let value = 0; let scale = 1; - for (;;) { + for (let digits = 1; ; digits += 1) { if (position >= mappings.length) refuse('MAPPINGS_TRUNCATED'); const code = mappings.charCodeAt(position); const digit = code < 128 ? DIGITS[code] : -1; @@ -108,6 +111,9 @@ export function decodeMappings(mappings, { sources, names }) { value += (digit & 31) * scale; if (value > VLQ_MAX) refuse('MAPPINGS_RANGE'); if ((digit & 32) === 0) break; + // A 32-bit value never needs more than seven digits; zero-valued padding beyond them would + // otherwise grow the scale to Infinity and the value to NaN, which no comparison refuses. + if (digits === MAX_DIGITS) refuse('MAPPINGS_RANGE'); scale *= 32; } const magnitude = Math.floor(value / 2); diff --git a/tools/cli-package/source-map.test.mjs b/tools/cli-package/source-map.test.mjs index 043554de..ea1f6b81 100644 --- a/tools/cli-package/source-map.test.mjs +++ b/tools/cli-package/source-map.test.mjs @@ -232,6 +232,11 @@ test('refuses every malformed mapping with its own code', () => { assert.throws(() => decodeMappings(mappings, counts), refuses(code), mappings); } assert.throws(() => decodeMappings(undefined, COUNTS), refuses('MAPPINGS_TYPE')); + assert.throws(() => decodeMappings('!', COUNTS), { + name: 'SourceMapError', + code: 'MAPPINGS_CHARACTER', + message: 'source map refused: MAPPINGS_CHARACTER', + }); // A refusal never quotes the mappings it read. assert.throws( () => decodeMappings('secretAAAA!', COUNTS), @@ -239,6 +244,34 @@ test('refuses every malformed mapping with its own code', () => { ); }); +test('bounds every value at exactly 0xffffffff in at most seven digits, and only base64 decodes', () => { + // The largest admitted value, 0xffffffff, is an original-line delta of -(2^31 - 1). + const back = `AA${vlq(2 ** 31 - 1)}A,CA${vlq(-(2 ** 31 - 1))}A`; + assert.equal(decodeMappings(back, COUNTS)[0][1].originalLine, 0); + // Seven digits decode; zero-payload padding past them refuses rather than growing the scale to + // Infinity, which would decode NaN and pass every later comparison. + assert.equal(decodeMappings(`${'g'.repeat(6)}A`, COUNTS)[0][0].generatedColumn, 0); + for (const [label, mappings, counts = COUNTS] of [ + ['one past the bound', vlq(2 ** 31)], + ['eight digits', `${'g'.repeat(7)}A`], + ['seven full digits and more', `${'/'.repeat(7)}${'g'.repeat(198)}A`], + ['a padded generated column', `${'g'.repeat(205)}A`], + ['a padded original column', `AAA${'g'.repeat(205)}A`], + ['a padded name index', `AAAA${'g'.repeat(205)}A`, { sources: 1, names: 0 }], + ]) { + assert.throws(() => decodeMappings(mappings, counts), refuses('MAPPINGS_RANGE'), label); + } + // DEL and the first non-ASCII code unit are no digits. + for (const code of [0x7f, 0x80]) { + assert.throws( + () => decodeMappings(String.fromCharCode(code), COUNTS), + refuses('MAPPINGS_CHARACTER'), + ); + } + // A column behind the previous segment refuses even when it is ahead of the line's first. + assert.throws(() => decodeMappings('A,K,F', COUNTS), refuses('MAPPINGS_ORDER')); +}); + test('finds only the segment that starts exactly at a position', () => { const lines = decodeMappings('A,EAAE,KAEC;;GAAA', COUNTS); assert.equal(segmentAt(lines, 0, 0).generatedColumn, 0); @@ -256,11 +289,17 @@ test('finds only the segment that starts exactly at a position', () => { ]) { assert.equal(segmentAt(lines, line, column), undefined, `${line}:${column}`); } + // A line's only segment is found exactly. + assert.equal(segmentAt(decodeMappings('A;;GAAA', COUNTS), 2, 3).generatedColumn, 3); }); test('counts lines and UTF-16 columns as V8 and TypeScript do', () => { assert.deepEqual(lineLengths('ab\ncd\r\nefg\rh
😀
'), [2, 2, 3, 1, 2, 0]); assert.deepEqual(lineLengths(''), [0]); + // U+2028 and U+2029 end a line; NEL (U+0085) does not. + const [nel, ls, ps] = [0x85, 0x2028, 0x2029].map((code) => String.fromCharCode(code)); + assert.deepEqual(lineLengths(`a${ls}b${ps}c`), [1, 1, 1]); + assert.deepEqual(lineLengths(`a${nel}b`), [3]); }); test('requires generated and original positions inside their texts', () => { @@ -276,4 +315,15 @@ test('requires generated and original positions inside their texts', () => { assert.throws(() => assertOriginalRange(lines, [[], []]), refuses('ORIGINAL_RANGE')); // Unmapped segments name no source and pass. assertOriginalRange(decodeMappings('A,C', COUNTS), []); + // Each original position is checked against its own source's lines, and a source with none + // refuses. + assertOriginalRange(decodeMappings('ACAA,CAAE', COUNTS), [[], [2]]); + assert.throws( + () => assertOriginalRange(decodeMappings('ACAE', COUNTS), [[5], [1]]), + refuses('ORIGINAL_RANGE'), + ); + assert.throws( + () => assertOriginalRange(decodeMappings('AAAA', COUNTS), []), + refuses('ORIGINAL_RANGE'), + ); }); diff --git a/tools/cli-package/symbolicate.mjs b/tools/cli-package/symbolicate.mjs index 47a432ac..ddf22c28 100644 --- a/tools/cli-package/symbolicate.mjs +++ b/tools/cli-package/symbolicate.mjs @@ -6,12 +6,13 @@ * frames through the staged final maps. Each anchor below must be reached by a synchronous frame that * lands exactly on a mapping segment and resolves to the call site that the matching source * revision's syntax tree gives: the candidate copy for a dirty local build, the checkout's Git objects - * at the recorded commit otherwise, both bound to the manifest's per-source digests. A refusal names a - * fixed code; no frame, path, position or source text is ever printed. + * at the recorded commit otherwise, both bound to the manifest's per-source digests. Each frame file is + * re-hashed after the run. A refusal names a fixed code; no frame, path, position or source text is + * ever printed. */ import { Buffer } from 'node:buffer'; import { randomBytes } from 'node:crypto'; -import { mkdirSync, realpathSync, writeFileSync } from 'node:fs'; +import { lstatSync, mkdirSync, realpathSync, writeFileSync } from 'node:fs'; import { createRequire } from 'node:module'; import { basename, dirname, join } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; @@ -81,6 +82,9 @@ function canaryDocument(canary) { /** * Run the finalized CLI's `import` of a canary workflow under the preload, with its home, project and * temporary files in `work` (owner-only, outside the candidate), and return the one captured stack. + * The project's own `.relavium/` exists before the run, so the CLI's upward search for a project stops + * there and never adopts an ancestor's — a developer's real one on Windows, where the temporary + * directory lies below the home; the run must have imported the canary into it. * @param {{candidate: string, work: string, run?: typeof runCommand}} options * @returns {Promise} */ @@ -88,7 +92,8 @@ export async function captureFrames({ candidate, work, run = runCommand }) { const canary = `frames-canary-${randomBytes(6).toString('hex')}`; const home = join(work, 'home'); const project = join(work, 'project'); - for (const directory of [home, project]) mkdirSync(directory, { mode: 0o700 }); + const catalog = join(project, '.relavium'); + for (const directory of [home, project, catalog]) mkdirSync(directory, { mode: 0o700 }); const document = join(work, `${canary}.relavium.yaml`); writeFileSync(document, canaryDocument(canary), { mode: 0o600, flag: 'wx' }); const log = join(work, 'frames.json'); @@ -119,6 +124,13 @@ export async function captureFrames({ candidate, work, run = runCommand }) { // The child's own output is never printed: it can quote the candidate. refuse('CAPTURE_FAILED'); } + let imported = false; + try { + imported = lstatSync(join(catalog, 'workflows', `${canary}.relavium.yaml`)).isFile(); + } catch { + imported = false; + } + if (!imported) refuse('CAPTURE_PROJECT'); let record; try { record = JSON.parse(readOrdinaryFile(log, 1024 * 1024).toString('utf8')); @@ -153,7 +165,8 @@ export async function captureFrames({ candidate, work, run = runCommand }) { /** * The text of each path at the matching source revision, bound to the manifest's digest: the * candidate copy for a dirty local build, otherwise the blob at the recorded commit, read from the - * original checkout's object store by Git without configuration, filters or replace refs. + * original checkout's object store by Git without home, global or system configuration, without + * filters or text conversion (`cat-file blob`) and without replace refs. * @param {{candidate: string, checkout: string, manifest: import('./manifest.mjs').PrivateManifest, * paths: readonly string[], run?: typeof runCommand}} options * @returns {Promise>} @@ -210,6 +223,7 @@ export function anchorPosition(ts, text, anchor) { const visit = (node) => { if ( (ts.isFunctionDeclaration(node) || ts.isMethodDeclaration(node)) && + node.body !== undefined && node.name !== undefined && node.name.getText(file) === anchor.within ) { @@ -265,7 +279,9 @@ export async function symbolicateCandidate({ revision = readSourceRevision, anchors = ANCHORS, }) { - if (anchors.length === 0) refuse('ANCHORS'); + // The maintainer's coverage ruling: at least two `@relavium/core` and three CLI call sites. + const within = (root) => anchors.filter((anchor) => anchor.source.startsWith(root)).length; + if (within('packages/core/src/') < 2 || within('apps/cli/src/') < 3) refuse('ANCHORS'); const manifest = readPrivateManifest(staging); const frames = await capture({ candidate, work, run }); const paths = [...new Set(anchors.map((anchor) => anchor.source))]; diff --git a/tools/cli-package/symbolicate.test.mjs b/tools/cli-package/symbolicate.test.mjs index 1e8cc604..199c504d 100644 --- a/tools/cli-package/symbolicate.test.mjs +++ b/tools/cli-package/symbolicate.test.mjs @@ -1,27 +1,38 @@ import assert from 'node:assert/strict'; import { Buffer } from 'node:buffer'; import { spawnSync } from 'node:child_process'; -import { createHash } from 'node:crypto'; import { - chmodSync, existsSync, mkdirSync, - mkdtempSync, readFileSync, readdirSync, - realpathSync, rmSync, statSync, symlinkSync, writeFileSync, } from 'node:fs'; -import { createRequire } from 'node:module'; -import { tmpdir } from 'node:os'; -import { dirname, join, posix } from 'node:path'; -import { after, test } from 'node:test'; -import { fileURLToPath } from 'node:url'; -import { MANIFEST_INPUTS, ManifestError, sealPrivateManifest } from './manifest.mjs'; -import { mapTrailer } from './finalize.mjs'; +import { dirname, join } from 'node:path'; +import { test } from 'node:test'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { + COMMIT, + PRELOAD, + SOURCES, + candidate, + cliRequire, + coreChunk, + digest, + fresh, + posixModes, + put, + reseal, + scratch, + seal, + ts, +} from './candidate-fixture.mjs'; +import * as keyringStub from './frame-capture-keyring.mjs'; +import { ManifestError, sealPrivateManifest } from './manifest.mjs'; +import { isolatedEnvironment, isolatedGitEnvironment, runCommand } from './process.mjs'; import { ANCHORS, SymbolicationError, @@ -31,205 +42,8 @@ import { symbolicateCandidate, } from './symbolicate.mjs'; -// A synthetic candidate whose TypeScript sits at the anchors' real repository paths, with the same -// function names, bundled by the exact esbuild behind the installed tsup with splitting and minify -// as the CLI build does. It runs under the real preload, so capture, maps and anchors are all real. -const cliRequire = createRequire(new URL('../../apps/cli/package.json', import.meta.url)); -const { build } = createRequire(cliRequire.resolve('tsup/package.json'))('esbuild'); -const ts = cliRequire('typescript'); -const cliModules = fileURLToPath(new URL('../../apps/cli/node_modules', import.meta.url)); -const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-symbolicate-'))); -after(() => rmSync(scratch, { recursive: true, force: true })); -const digest = (bytes) => createHash('sha256').update(bytes).digest('hex'); -const posixModes = process.platform !== 'win32'; -let count = 0; - -const SOURCES = { - 'packages/core/src/yaml-decode.ts': [ - "import { parse as parseYaml } from 'yaml';", - '', - 'export function decodeHardenedYaml(text: string): unknown {', - " return parseYaml(text, { version: '1.2' });", - '}', - '', - ], - 'packages/core/src/parser.ts': [ - "import { decodeHardenedYaml } from './yaml-decode.js';", - '', - 'export function parseWorkflow(text: string): { id: string } {', - ' const raw = decodeHardenedYaml(text) as { workflow: { id: string } };', - ' return { id: raw.workflow.id };', - '}', - '', - ], - 'apps/cli/src/authoring/authoring.ts': [ - "import { parseWorkflow } from '../../../../packages/core/src/parser.js';", - '', - 'export function detectAndParse(text: string, fileName: string): { id: string } {', - " if (fileName.endsWith('.relavium.yaml')) {", - ' return parseAsWorkflow(text);', - ' }', - " throw new Error('unsupported');", - '}', - '', - 'function parseAsWorkflow(text: string): { id: string } {', - ' return parseWorkflow(text);', - '}', - '', - ], - 'apps/cli/src/commands/import.ts': [ - "import { readFileSync } from 'node:fs';", - "import { basename } from 'node:path';", - "import { detectAndParse } from '../authoring/authoring.js';", - '', - 'export function importCommand(path: string): string {', - " const text = readFileSync(path, 'utf8');", - ' return detectAndParse(text, basename(path)).id;', - '}', - '', - ], - 'apps/cli/src/index.ts': [ - '#!/usr/bin/env node', - "import { Entry } from '@napi-rs/keyring';", - 'try {', - " new Entry('relavium', 'probe');", - '} catch {', - ' // The preload replaces the keyring binding with one that refuses.', - '}', - 'const [command, path] = process.argv.slice(2);', - "if (command === 'import' && path !== undefined) {", - " const { importCommand } = await import('./commands/import.js');", - ' importCommand(path);', - '}', - '', - ], -}; - -/** Write `path` (repository-relative) under `root`. */ -function put(root, path, bytes) { - mkdirSync(dirname(join(root, ...path.split('/'))), { recursive: true }); - writeFileSync(join(root, ...path.split('/')), bytes); -} - -/** - * A finalized synthetic candidate: its finalized dist without trailers, its staged maps with - * repository-relative sources, the finalizer's record, the manifest inputs and the CLI installation. - */ -async function candidate({ sources = SOURCES } = {}) { - count += 1; - const root = join(scratch, `candidate-${count}`); - const staging = join(scratch, `staging-${count}`); - const work = join(scratch, `work-${count}`); - for (const directory of [staging, work]) { - mkdirSync(directory); - chmodSync(directory, 0o700); - } - mkdirSync(join(staging, 'maps')); - for (const [path, lines] of Object.entries(sources)) put(root, path, lines.join('\n')); - for (const path of MANIFEST_INPUTS) { - put( - root, - path, - path.endsWith('.json') ? `${JSON.stringify({ synthetic: path })}\n` : `# ${path}\n`, - ); - } - put(root, 'apps/cli/tsup.config.ts', "export default {\n target: 'node22',\n};\n"); - put(root, 'apps/cli/package.json', JSON.stringify({ name: 'relavium', type: 'module' })); - symlinkSync(cliModules, join(root, 'apps', 'cli', 'node_modules'), 'junction'); - const cli = join(root, 'apps', 'cli'); - const built = await build({ - absWorkingDir: cli, - entryPoints: { index: 'src/index.ts' }, - outdir: 'dist', - format: 'esm', - platform: 'node', - target: 'node22', - bundle: true, - splitting: true, - minify: true, - sourcemap: 'external', - sourcesContent: false, - external: ['yaml', '@napi-rs/keyring'], - write: false, - logLevel: 'silent', - }); - const outputs = []; - const maps = []; - for (const file of built.outputFiles) { - const name = posix.basename(file.path.split('\\').join('/')); - if (name.endsWith('.map')) { - const map = JSON.parse(file.text); - map.sources = map.sources.map((source) => - posix.normalize(posix.join('apps/cli/dist', source)), - ); - const bytes = Buffer.from(JSON.stringify(map)); - writeFileSync(join(staging, 'maps', name), bytes); - maps.push({ path: `dist/${name}`, bytes: bytes.length, sha256: digest(bytes) }); - } else { - // An external map leaves no link in the code: this is already the finalized JS. - const code = file.text; - assert.equal(code.includes(mapTrailer(name)), false); - put(cli, `dist/${name}`, code); - const bytes = Buffer.from(code); - outputs.push({ path: `dist/${name}`, bytes: bytes.length, sha256: digest(bytes) }); - } - } - writeFileSync( - join(staging, 'finalized.json'), - JSON.stringify({ version: 1, outputs, maps, upstreamMaps: 0 }), - ); - const archive = { - sha256: 'a'.repeat(64), - package: { version: '0.1.1' }, - files: outputs.map((output) => ({ path: `package/${output.path}`, sha256: output.sha256 })), - }; - return { root, cli, staging, work, archive, outputs }; -} - -const TREE = 'b'.repeat(40); -const COMMIT = 'c'.repeat(40); -/** Git as the manifest asks it: the commit's tree. */ -const git = async (command, args) => { - assert.equal(command, 'git'); - assert.deepEqual(args, ['--no-replace-objects', 'rev-parse', '--verify', `${COMMIT}^{tree}`]); - return { stdout: `${TREE}\n`, stderr: '' }; -}; - -/** Seal the manifest of a dirty local candidate, whose source revision is its own copy. */ -async function seal(fixture) { - return sealPrivateManifest({ - candidate: fixture.root, - staging: fixture.staging, - checkout: fixture.root, - source: COMMIT, - dirty: true, - archive: fixture.archive, - run: git, - }); -} - -/** Re-record the finalized digests after a deliberate change, then seal a fresh manifest. */ -async function reseal(fixture) { - const outputs = readdirSync(join(fixture.cli, 'dist')).map((name) => { - const bytes = readFileSync(join(fixture.cli, 'dist', name)); - return { path: `dist/${name}`, bytes: bytes.length, sha256: digest(bytes) }; - }); - const maps = readdirSync(join(fixture.staging, 'maps')).map((name) => { - const bytes = readFileSync(join(fixture.staging, 'maps', name)); - return { path: `dist/${name}`, bytes: bytes.length, sha256: digest(bytes) }; - }); - rmSync(join(fixture.staging, 'finalized.json')); - rmSync(join(fixture.staging, 'manifest.json'), { force: true }); - writeFileSync( - join(fixture.staging, 'finalized.json'), - JSON.stringify({ version: 1, outputs, maps, upstreamMaps: 0 }), - ); - fixture.archive.files = outputs.map((output) => ({ - path: `package/${output.path}`, - sha256: output.sha256, - })); - return seal(fixture); -} +const refuses = (code) => (error) => error instanceof SymbolicationError && error.code === code; +const repository = fileURLToPath(new URL('../../', import.meta.url)); function symbolicate(fixture, options = {}) { return symbolicateCandidate({ @@ -242,114 +56,31 @@ function symbolicate(fixture, options = {}) { }); } -const refuses = (code) => (error) => error instanceof SymbolicationError && error.code === code; - -/** The finalized chunk that holds the core call sites. */ -function coreChunk(fixture) { - return readdirSync(join(fixture.cli, 'dist')).find((name) => - JSON.parse(readFileSync(join(fixture.staging, 'maps', `${name}.map`), 'utf8')).sources.includes( - 'packages/core/src/parser.ts', - ), - ); +/** A sealed fixture and the frames one real capture of it produced. */ +async function sealedWithFrames() { + const fixture = await candidate(); + await seal(fixture); + const frames = await captureFrames({ candidate: fixture.root, work: fixture.work }); + return { fixture, frames }; } -test('the manifest binds the commit, toolchain, inputs, finalized pairs, sources and tarball', async () => { - const fixture = await candidate(); - const manifest = await seal(fixture); - assert.deepEqual(manifest.source, { commit: COMMIT, tree: TREE, dirty: true }); - assert.equal(manifest.packageVersion, '0.1.1'); - assert.equal(manifest.target, 'node22'); - assert.equal(manifest.tarballSha256, 'a'.repeat(64)); - assert.equal(manifest.toolchain.node, process.version); - assert.equal(manifest.toolchain.typescript, ts.version); - assert.deepEqual(Object.keys(manifest.inputs), [...MANIFEST_INPUTS]); - assert.equal( - manifest.inputs['apps/cli/tsup.config.ts'], - digest(readFileSync(join(fixture.cli, 'tsup.config.ts'))), - ); - assert.deepEqual( - manifest.outputs.map((output) => output.path), - fixture.outputs.map((output) => output.path).sort(), - ); +/** Re-record the manifest's map digest for `chunk` after a deliberate change to its staged map. */ +function rerecordManifestMap(fixture, chunk) { + const path = join(fixture.staging, 'manifest.json'); + const manifest = JSON.parse(readFileSync(path, 'utf8')); + const bytes = readFileSync(join(fixture.staging, 'maps', `${chunk}.map`)); for (const output of manifest.outputs) { - assert.equal(output.sha256, digest(readFileSync(join(fixture.cli, output.path)))); - assert.deepEqual(output.map, { - path: `${output.path}.map`, - sha256: digest( - readFileSync(join(fixture.staging, 'maps', `${posix.basename(output.path)}.map`)), - ), - }); + if (output.path === `dist/${chunk}`) output.map.sha256 = digest(bytes); } - assert.deepEqual(Object.keys(manifest.sources), Object.keys(SOURCES).sort()); - for (const [path, value] of Object.entries(manifest.sources)) { - assert.equal(value, digest(readFileSync(join(fixture.root, ...path.split('/'))))); - } - // Written once, owner-only, and holding no absolute path. - const text = readFileSync(join(fixture.staging, 'manifest.json'), 'utf8'); - assert.deepEqual(JSON.parse(text), manifest); - assert.equal(text.includes(scratch), false); - if (posixModes) { - assert.equal(statSync(join(fixture.staging, 'manifest.json')).mode & 0o777, 0o600); - } - await assert.rejects(seal(fixture), (error) => error.code === 'EEXIST'); -}); + writeFileSync(path, JSON.stringify(manifest, null, 2) + '\n'); +} -test('the manifest refuses drifted maps and tarball entries and a missing tree', async () => { - for (const [label, change, code] of [ - [ - 'a staged map changed after the finalizer recorded it', - (fixture) => - writeFileSync(join(fixture.staging, 'maps', `${coreChunk(fixture)}.map`), '{"sources":[]}'), - 'MAP_DRIFT', - ], - [ - 'a tarball entry that is not the finalized JS', - (fixture) => (fixture.archive.files[0].sha256 = 'd'.repeat(64)), - 'TARBALL_DRIFT', - ], - [ - 'an output without its map in the record', - (fixture) => { - const record = JSON.parse(readFileSync(join(fixture.staging, 'finalized.json'), 'utf8')); - record.maps.pop(); - writeFileSync(join(fixture.staging, 'finalized.json'), JSON.stringify(record)); - }, - 'RECORD', - ], - [ - 'a build configuration without its target', - (fixture) => writeFileSync(join(fixture.cli, 'tsup.config.ts'), 'export default {};\n'), - 'TARGET', - ], - [ - 'a missing manifest input', - (fixture) => rmSync(join(fixture.root, 'pnpm-lock.yaml')), - 'UNREADABLE_INPUT', - ], - ]) { - const fixture = await candidate(); - change(fixture); - await assert.rejects( - seal(fixture), - (error) => error instanceof ManifestError && error.code === code, - label, - ); - assert.equal(existsSync(join(fixture.staging, 'manifest.json')), false, label); - } - const fixture = await candidate(); - await assert.rejects( - sealPrivateManifest({ - candidate: fixture.root, - staging: fixture.staging, - checkout: fixture.root, - source: COMMIT, - dirty: false, - archive: fixture.archive, - run: async () => ({ stdout: 'not a tree\n', stderr: '' }), - }), - (error) => error instanceof ManifestError && error.code === 'SOURCE_TREE', - ); -}); +/** Entry code that imports the canary into the run's project, as the real CLI does. */ +const IMPORTS = + "import { mkdirSync as m, writeFileSync as w } from 'node:fs'; import { join as j } from 'node:path';\n" + + 'const [, , , project] = process.argv.slice(2);\n' + + "m(j(project, '.relavium', 'workflows'), { recursive: true });\n" + + "w(j(project, '.relavium', 'workflows', `${process.env.RELAVIUM_FRAME_CAPTURE_CANARY}.relavium.yaml`), '');\n"; test('actual packed frames reach every anchor through the real preload and exact segments', async () => { const fixture = await candidate(); @@ -359,16 +90,108 @@ test('actual packed frames reach every anchor through the real preload and exact capture: async (options) => (frames = await captureFrames(options)), }); assert.deepEqual(result, { anchors: ANCHORS.length }); - // One stack crossed the entry, a lazy chunk and the inlined core. + // One stack crossed the entry and the lazy chunk that inlines the core. const files = new Set( - frames - .filter((frame) => frame.file?.includes('/dist/') || frame.file?.includes('\\dist\\')) - .map((frame) => frame.file), + frames.filter((frame) => frame.file?.includes('/dist/')).map((frame) => frame.file), ); assert.ok(files.size >= 2, 'the stack crosses more than one finalized chunk'); - // The run kept its files in the private work directory. - assert.equal(existsSync(join(fixture.work, 'frames.json')), true); + // The canary was imported into the run's own project, never the candidate. + const imported = readdirSync(join(fixture.work, 'project', '.relavium', 'workflows')); + assert.equal(imported.length, 1); + assert.match(imported[0], /^frames-canary-[0-9a-f]{12}\.relavium\.yaml$/u); assert.equal(existsSync(join(fixture.root, '.relavium')), false); + // The count is the table's, not a constant. + assert.deepEqual( + await symbolicate(fixture, { anchors: [...ANCHORS, ANCHORS[0]], capture: async () => frames }), + { anchors: ANCHORS.length + 1 }, + ); +}); + +test('an ancestor project of the work directory is never adopted by the canary import', async () => { + // A developer's real `.relavium/` above the temporary directory, as on Windows. + const ancestor = fresh('ancestor'); + mkdirSync(join(ancestor, '.relavium'), { recursive: true }); + const fixture = await candidate({ parent: ancestor }); + await seal(fixture); + assert.deepEqual(await symbolicate(fixture), { anchors: ANCHORS.length }); + assert.deepEqual(readdirSync(join(ancestor, '.relavium')), []); + assert.equal(readdirSync(join(fixture.work, 'project', '.relavium', 'workflows')).length, 1); + // A run that imports nothing into its project refuses. + const elsewhere = await candidate(); + writeFileSync( + join(elsewhere.cli, 'dist', 'index.js'), + "import { readFileSync } from 'node:fs'; import { parse } from 'yaml';\n" + + "parse(readFileSync(process.argv[3], 'utf8'));\n", + ); + await assert.rejects( + captureFrames({ candidate: elsewhere.root, work: elsewhere.work }), + refuses('CAPTURE_PROJECT'), + ); + // So does one that writes its project's catalog but not the canary. + const other = fresh('other-import'); + mkdirSync(other, { mode: 0o700 }); + await assert.rejects( + captureFrames({ + candidate: scratch, + work: other, + run: async (_command, args, options) => { + const workflows = join(args.at(-1), '.relavium', 'workflows'); + mkdirSync(workflows); + writeFileSync(join(workflows, 'another.relavium.yaml'), ''); + writeFileSync( + options.env.RELAVIUM_FRAME_CAPTURE_LOG, + JSON.stringify({ + version: 1, + captures: [[{ file: null, line: null, column: null, async: false }]], + }), + ); + return { stdout: '', stderr: '' }; + }, + }), + refuses('CAPTURE_PROJECT'), + ); +}); + +test('the anchors keep the coverage ruling and resolve in the real repository', async () => { + const core = ANCHORS.filter((anchor) => anchor.source.startsWith('packages/core/src/')); + const cli = ANCHORS.filter((anchor) => anchor.source.startsWith('apps/cli/src/')); + assert.ok(core.length >= 2, 'at least two core anchors'); + assert.ok(cli.length >= 3, 'at least three CLI anchors'); + assert.equal(Object.isFrozen(ANCHORS), true); + for (const anchor of ANCHORS) { + const text = readFileSync(join(repository, ...anchor.source.split('/')), 'utf8'); + const position = anchorPosition(ts, text, anchor); + const line = text.split(/\r\n|[\n\r\u2028\u2029]/u)[position.line - 1]; + assert.equal(line.slice(position.column - 1).startsWith(anchor.call), true, anchor.within); + } + // A table below the ruling refuses before anything runs. + const fixture = await candidate(); + await seal(fixture); + for (const anchors of [ + [], + ANCHORS.slice(1), + ANCHORS.slice(0, 4), + ANCHORS.filter((anchor) => !anchor.source.startsWith('packages/core/')), + // Call sites in other engine packages, or outside a package's sources, are not core anchors. + [ + ...cli, + { source: 'packages/shared/src/a.ts', within: 'f', call: 'g' }, + { source: 'packages/core/test/b.ts', within: 'f', call: 'g' }, + ], + ]) { + let ran = false; + await assert.rejects( + symbolicate(fixture, { + anchors, + capture: async () => { + ran = true; + return []; + }, + }), + refuses('ANCHORS'), + ); + assert.equal(ran, false); + } }); test('the expected positions come from the source revision, independently of any map', () => { @@ -378,26 +201,50 @@ test('the expected positions come from the source revision, independently of any const line = text.split('\n')[position.line - 1]; assert.equal(line.slice(position.column - 1).startsWith(anchor.call), true, anchor.call); } - // A member call is positioned at its property name; a second call or owner is ambiguous. + const at = (text, within = 'f', call = 'g') => + anchorPosition(ts, text, { source: 'a.ts', within, call }); + // A member call is positioned at its property name. + assert.deepEqual(at('function f() {\n yaml.parse(x);\n}\n', 'f', 'parse'), { + line: 2, + column: 8, + }); + // Declaration, arrow, function expression and method owners; a call outside the owner is ignored. + assert.deepEqual(at('function f() { g(); }\nfunction h() { g(); }'), { line: 1, column: 16 }); + assert.deepEqual(at('const f = () => {\n g();\n};'), { line: 2, column: 3 }); + assert.deepEqual(at('const f = function () {\n g();\n};'), { line: 2, column: 3 }); + assert.deepEqual(at('class A {\n f() {\n g();\n }\n}'), { line: 3, column: 5 }); + assert.deepEqual(at('const f = () => g();\nconst h = () => g();'), { line: 1, column: 17 }); + assert.deepEqual(at('let f;\nfunction f() { g(); }'), { line: 2, column: 16 }); + assert.deepEqual(at('export default function () { g(); }\nfunction f() { g(); }'), { + line: 2, + column: 16, + }); + // Bodiless signatures (overloads, declarations) are not owners; the implementation is. assert.deepEqual( - anchorPosition(ts, 'function f() {\n yaml.parse(x);\n}\n', { - source: 'a.ts', - within: 'f', - call: 'parse', - }), - { line: 2, column: 8 }, + at( + 'function f(a: string): void;\nfunction f(a: number): void;\nfunction f(a: unknown) { g(); }', + ), + { line: 3, column: 26 }, ); + assert.deepEqual(at('declare function f(): void;\nfunction f() { g(); }'), { + line: 2, + column: 16, + }); + // A missing owner, a second owner, no call or a second call is ambiguous. for (const text of [ + 'function h() { g(); }', 'function f() { g(); g(); }', 'function f() { h(); }', 'function f() { g(); }\nfunction f() { g(); }', ]) { - assert.throws( - () => anchorPosition(ts, text, { source: 'a.ts', within: 'f', call: 'g' }), - refuses('ANCHOR_SOURCE'), - text, - ); + assert.throws(() => at(text), refuses('ANCHOR_SOURCE'), text); } + // A refusal names only its fixed code. + assert.throws(() => at('function h() {}'), { + name: 'SymbolicationError', + code: 'ANCHOR_SOURCE', + message: 'PACKED_CLI_SYMBOLICATION_ANCHOR_SOURCE', + }); }); test('wrong JS, a wrong map, permuted sources and a wrong source revision cannot reach the anchors', async () => { @@ -442,6 +289,18 @@ test('wrong JS, a wrong map, permuted sources and a wrong source revision cannot writeFileSync(path, `// shifted\n${readFileSync(path, 'utf8')}`); }, ], + [ + 'a source revision one column apart', + (fixture) => { + const path = join(fixture.root, 'packages', 'core', 'src', 'parser.ts'); + const text = readFileSync(path, 'utf8'); + assert.ok(text.includes('const raw = decodeHardenedYaml')); + writeFileSync( + path, + text.replace('const raw = decodeHardenedYaml', 'const raw = decodeHardenedYaml'), + ); + }, + ], ]) { const fixture = await candidate(); change(fixture); @@ -450,15 +309,31 @@ test('wrong JS, a wrong map, permuted sources and a wrong source revision cannot } // Another chunk's map names other sources, so the resealed manifest no longer covers the anchors. const fixture = await candidate(); - const core = `${coreChunk(fixture)}.map`; writeFileSync( - join(fixture.staging, 'maps', core), + join(fixture.staging, 'maps', `${coreChunk(fixture)}.map`), readFileSync(join(fixture.staging, 'maps', 'index.js.map')), ); await reseal(fixture); await assert.rejects(symbolicate(fixture), refuses('SOURCE_REVISION')); }); +test('a call after non-ASCII text on its line still reaches its anchor', async () => { + const sources = { + ...SOURCES, + 'packages/core/src/yaml-decode.ts': [ + "import { parse as parseYaml } from 'yaml';", + '', + 'export function decodeHardenedYaml(text: string): unknown {', + " const mark = 'é—😀'; return mark.length > 0 ? parseYaml(text, { version: '1.2' }) : null;", + '}', + '', + ], + }; + const fixture = await candidate({ sources }); + await seal(fixture); + assert.deepEqual(await symbolicate(fixture), { anchors: ANCHORS.length }); +}); + test('files that are not the manifest’s refuse by their digest', async () => { for (const [label, change, code] of [ [ @@ -469,6 +344,26 @@ test('files that are not the manifest’s refuse by their digest', async () => { }, 'FRAME_FILE', ], + [ + 'finalized JS the tarball agrees with but the manifest does not', + (fixture) => { + const chunk = coreChunk(fixture); + const path = join(fixture.cli, 'dist', chunk); + writeFileSync(path, `${readFileSync(path, 'utf8')} `); + fixture.archive.files.find((file) => file.path === `package/dist/${chunk}`).sha256 = digest( + readFileSync(path), + ); + }, + 'FRAME_FILE', + ], + [ + 'a tarball entry that is not the run file', + (fixture) => { + const key = `package/dist/${coreChunk(fixture)}`; + fixture.archive.files.find((file) => file.path === key).sha256 = 'e'.repeat(64); + }, + 'FRAME_FILE', + ], [ 'a staged map changed after the manifest', (fixture) => { @@ -485,43 +380,123 @@ test('files that are not the manifest’s refuse by their digest', async () => { }, 'SOURCE_REVISION', ], - [ - 'a tarball entry that is not the run file', - (fixture) => { - const key = `package/dist/${coreChunk(fixture)}`; - fixture.archive.files.find((file) => file.path === key).sha256 = 'e'.repeat(64); - }, - 'FRAME_FILE', - ], ]) { const fixture = await candidate(); await seal(fixture); change(fixture); await assert.rejects(symbolicate(fixture), refuses(code), label); } - const fixture = await candidate(); - await assert.rejects(symbolicate(fixture), (error) => error instanceof ManifestError); - await seal(fixture); - await assert.rejects(symbolicate(fixture, { anchors: [] }), refuses('ANCHORS')); + // Without a manifest the candidate is never run. + const bare = await candidate(); + let ran = false; + await assert.rejects( + symbolicate(bare, { + capture: async () => { + ran = true; + return []; + }, + }), + (error) => error instanceof ManifestError && error.code === 'UNREADABLE', + ); + assert.equal(ran, false); }); -test('a clean candidate reads its source revision from the checkout’s Git objects', async () => { - const checkout = join(scratch, 'checkout'); - mkdirSync(checkout); - const gitRun = (args) => { - const run = spawnSync('git', args, { - cwd: checkout, - encoding: 'utf8', - env: { PATH: process.env.PATH, HOME: checkout, GIT_CONFIG_NOSYSTEM: '1' }, - }); - assert.equal(run.status, 0, run.stderr); - return run.stdout.trim(); - }; - gitRun(['init', '--quiet']); - const path = 'apps/cli/src/commands/import.ts'; - put(checkout, path, SOURCES[path].join('\n')); - gitRun(['add', '.']); - gitRun([ +test('a recorded map that does not decode refuses with its own code', async () => { + for (const [label, change] of [ + ['not JSON', () => '{'], + ['malformed mappings', (map) => JSON.stringify({ ...map, mappings: 'A!' })], + [ + 'a source index out of range', + (map) => JSON.stringify({ ...map, sources: map.sources.slice(0, -1) }), + ], + [ + 'a name index out of range', + (map) => JSON.stringify({ ...map, names: map.names.slice(0, -1) }), + ], + ]) { + const fixture = await candidate(); + await seal(fixture); + const chunk = coreChunk(fixture); + const path = join(fixture.staging, 'maps', `${chunk}.map`); + const map = JSON.parse(readFileSync(path, 'utf8')); + assert.ok(map.names.length > 0, 'the minified chunk records names'); + writeFileSync(path, change(map)); + rerecordManifestMap(fixture, chunk); + await assert.rejects(symbolicate(fixture), refuses('MAP_DECODE'), label); + } +}); + +test('only synchronous frames inside the finalized dist count, and an unrecorded one refuses', async () => { + const { fixture, frames } = await sealedWithFrames(); + assert.deepEqual(await symbolicate(fixture, { capture: async () => frames }), { + anchors: ANCHORS.length, + }); + // Anchors reached only by asynchronous frames do not count. + await assert.rejects( + symbolicate(fixture, { + capture: async () => frames.map((frame) => ({ ...frame, async: true })), + }), + refuses('ANCHOR_UNMAPPED'), + ); + // Native, node-internal, preload and sibling-directory frames are skipped. + const sibling = join(fixture.cli, 'distx'); + mkdirSync(sibling); + writeFileSync(join(sibling, 'a.js'), ''); + const skipped = [ + { file: null, line: null, column: null, async: false }, + { file: 'node:internal/modules/run_main', line: 1, column: 1, async: false }, + { file: pathToFileURL(PRELOAD).href, line: 1, column: 1, async: false }, + { file: pathToFileURL(join(sibling, 'a.js')).href, line: 1, column: 1, async: false }, + ]; + assert.deepEqual(await symbolicate(fixture, { capture: async () => [...skipped, ...frames] }), { + anchors: ANCHORS.length, + }); + // A dist frame the manifest does not record, or that cannot be resolved, refuses. + writeFileSync(join(fixture.cli, 'dist', 'extra.js'), ''); + for (const name of ['extra.js', 'missing.js']) { + const frame = { + file: pathToFileURL(join(fixture.cli, 'dist', name)).href, + line: 1, + column: 1, + async: false, + }; + await assert.rejects( + symbolicate(fixture, { capture: async () => [frame, ...frames] }), + refuses('FRAME_FILE'), + name, + ); + } + // A symlinked candidate path reaches the same dist. + const link = fresh('linked-candidate'); + symlinkSync(fixture.root, link, 'junction'); + assert.deepEqual(await symbolicate(fixture, { candidate: link, capture: async () => frames }), { + anchors: ANCHORS.length, + }); + // Without the candidate's TypeScript the run refuses with its code. A fresh candidate: Node's + // resolution cache would still answer for one that already resolved it. + const untooled = await candidate(); + await seal(untooled); + rmSync(join(untooled.cli, 'node_modules')); + await assert.rejects( + symbolicate(untooled, { capture: async () => frames }), + refuses('TOOLCHAIN'), + ); +}); + +/** Git without the host's configuration, for building test repositories. */ +function gitIn(directory, args) { + const run = spawnSync('git', args, { + cwd: directory, + encoding: 'utf8', + env: { PATH: process.env.PATH, HOME: directory, GIT_CONFIG_NOSYSTEM: '1' }, + }); + assert.equal(run.status, 0, run.stderr); + return run.stdout.trim(); +} + +function commitAll(directory) { + gitIn(directory, ['add', '.']); + gitIn(directory, [ '-c', 'user.name=t', '-c', @@ -531,40 +506,278 @@ test('a clean candidate reads its source revision from the checkout’s Git obje '-m', 'x', ]); - const commit = gitRun(['rev-parse', 'HEAD']); - const text = SOURCES[path].join('\n'); - const manifest = { source: { commit, dirty: false }, sources: { [path]: digest(text) } }; - const texts = await readSourceRevision({ - candidate: join(scratch, 'no-candidate'), + return gitIn(directory, ['rev-parse', 'HEAD']); +} + +test('a clean candidate reads the recorded commit from the checkout, never HEAD or its own copy', async () => { + const fixture = await candidate(); + const checkout = fresh('checkout'); + mkdirSync(checkout); + gitIn(checkout, ['init', '--quiet']); + for (const [path, lines] of Object.entries(SOURCES)) put(checkout, path, lines.join('\n')); + const recorded = commitAll(checkout); + put( checkout, - manifest, + 'packages/core/src/parser.ts', + `// later\n${SOURCES['packages/core/src/parser.ts'].join('\n')}`, + ); + commitAll(checkout); + await sealPrivateManifest({ + candidate: fixture.root, + staging: fixture.staging, + checkout, + source: recorded, + dirty: false, + archive: fixture.archive, + }); + // The candidate copy changes after the seal too: a clean build must not read it. + const copy = join(fixture.root, 'packages', 'core', 'src', 'yaml-decode.ts'); + writeFileSync(copy, `// later\n${readFileSync(copy, 'utf8')}`); + // The run port reaches the capture and each Git read, one per anchor source. + const commands = []; + const run = (command, args, options) => { + commands.push(command === 'git' ? args.slice(0, 3).join(' ') : command); + return runCommand(command, args, options); + }; + assert.deepEqual(await symbolicate(fixture, { checkout, run }), { anchors: ANCHORS.length }); + assert.deepEqual(commands, [ + process.execPath, + ...Array(new Set(ANCHORS.map((anchor) => anchor.source)).size).fill( + '--no-replace-objects cat-file blob', + ), + ]); +}); + +test('a dirty candidate reads its own copy, never the checkout', async () => { + const fixture = await candidate(); + await seal(fixture); + const empty = fresh('empty-checkout'); + mkdirSync(empty); + assert.deepEqual(await symbolicate(fixture, { checkout: empty }), { anchors: ANCHORS.length }); +}); + +test('the source revision is read without configuration and refuses with its fixed code', async () => { + const path = 'apps/cli/src/commands/import.ts'; + const text = 'é — 😀\nexport function f() {}\n'; + const clean = { + source: { commit: COMMIT, dirty: false }, + sources: { [path]: digest(Buffer.from(text, 'utf8')) }, + }; + const calls = []; + const texts = await readSourceRevision({ + candidate: fresh('nowhere'), + checkout: scratch, + manifest: clean, paths: [path], + run: async (command, args, options) => { + calls.push({ command, args, options }); + return { stdout: text, stderr: '' }; + }, }); + // Non-ASCII text round-trips. assert.equal(texts.get(path), text); - // A commit whose blob is not the built source refuses; so does a path outside the manifest. - await assert.rejects( - readSourceRevision({ - candidate: checkout, - checkout, - manifest: { ...manifest, sources: { [path]: digest(`${text}\n`) } }, - paths: [path], - }), - refuses('SOURCE_REVISION'), - ); - await assert.rejects( - readSourceRevision({ candidate: checkout, checkout, manifest, paths: ['apps/cli/src/x.ts'] }), - refuses('SOURCE_REVISION'), + assert.deepEqual( + calls.map(({ command, args, options }) => [command, args, options.cwd, options.env]), + [ + [ + 'git', + ['--no-replace-objects', 'cat-file', 'blob', `${COMMIT}:${path}`], + scratch, + isolatedGitEnvironment(), + ], + ], ); + let unlistedRead = false; + for (const [label, options] of [ + [ + 'a missing dirty copy', + { + candidate: fresh('nowhere'), + manifest: { source: { commit: COMMIT, dirty: true }, sources: { [path]: 'f'.repeat(64) } }, + }, + ], + [ + 'a failed Git read', + { + candidate: scratch, + manifest: clean, + run: async () => { + throw new Error('git failed'); + }, + }, + ], + [ + 'a blob that is not the built source', + { + candidate: scratch, + manifest: { ...clean, sources: { [path]: digest(`${text}\n`) } }, + run: async () => ({ stdout: text, stderr: '' }), + }, + ], + [ + 'a path outside the manifest, refused before Git runs', + { + candidate: scratch, + manifest: clean, + paths: ['apps/cli/src/x.ts'], + run: async () => { + unlistedRead = true; + return { stdout: text, stderr: '' }; + }, + }, + ], + ]) { + await assert.rejects( + readSourceRevision({ checkout: scratch, paths: [path], ...options }), + refuses('SOURCE_REVISION'), + label, + ); + } + assert.equal(unlistedRead, false); +}); + +test('the CLI runs in the private work directory with an isolated environment', async () => { + const fixture = await candidate(); + process.env.RELAVIUM_SYMBOLICATION_SENTINEL = 'ambient-secret'; + const calls = []; + try { + await captureFrames({ + candidate: fixture.root, + work: fixture.work, + run: (command, args, options) => { + calls.push({ command, args, options }); + return runCommand(command, args, options); + }, + }); + } finally { + delete process.env.RELAVIUM_SYMBOLICATION_SENTINEL; + } + const [{ command, args, options }] = calls; + assert.equal(calls.length, 1); + assert.equal(command, process.execPath); + const canary = options.env.RELAVIUM_FRAME_CAPTURE_CANARY; + assert.match(canary, /^frames-canary-[0-9a-f]{12}$/u); + const home = join(fixture.work, 'home'); + const project = join(fixture.work, 'project'); + const document = join(fixture.work, `${canary}.relavium.yaml`); + assert.deepEqual(args, [ + '--import', + pathToFileURL(PRELOAD).href, + join(fixture.root, 'apps', 'cli', 'dist', 'index.js'), + 'import', + document, + '--cwd', + project, + ]); + assert.equal(options.cwd, fixture.work); + assert.deepEqual(options.env, { + ...isolatedEnvironment(home), + RELAVIUM_FRAME_CAPTURE_LOG: join(fixture.work, 'frames.json'), + RELAVIUM_FRAME_CAPTURE_PACKAGE: join(fixture.root, 'apps', 'cli', 'package.json'), + RELAVIUM_FRAME_CAPTURE_CANARY: canary, + }); + if (posixModes) { + for (const directory of [home, project, join(project, '.relavium')]) { + assert.equal(statSync(directory).mode & 0o777, 0o700, directory); + } + assert.equal(statSync(document).mode & 0o777, 0o600); + } +}); + +test('the capture validates the record it reads', async () => { + const valid = { file: 'file:///x.js', line: 1, column: 1, async: false }; + const cases = [ + ['no record', undefined, 'CAPTURE_FAILED'], + [ + 'an oversized record', + `${JSON.stringify({ version: 1, captures: [[valid]] })}${' '.repeat(1024 * 1024)}`, + 'CAPTURE_FAILED', + ], + ['null', 'null', 'CAPTURE_SHAPE'], + ['another version', { version: 2, captures: [[valid]] }, 'CAPTURE_SHAPE'], + ['captures not an array', { version: 1, captures: 'xy' }, 'CAPTURE_SHAPE'], + ['captures array-like', { version: 1, captures: { 0: [valid], length: 1 } }, 'CAPTURE_SHAPE'], + ['no capture', { version: 1, captures: [] }, 'CAPTURE_COUNT'], + ['two captures', { version: 1, captures: [[valid], [valid]] }, 'CAPTURE_COUNT'], + ['a stack that is not an array', { version: 1, captures: [{}] }, 'CAPTURE_SHAPE'], + ['an empty stack', { version: 1, captures: [[]] }, 'CAPTURE_SHAPE'], + ['65 frames', { version: 1, captures: [Array(65).fill(valid)] }, 'CAPTURE_SHAPE'], + ['a null frame', { version: 1, captures: [[null]] }, 'CAPTURE_SHAPE'], + ['a numeric file', { version: 1, captures: [[{ ...valid, file: 5 }]] }, 'CAPTURE_SHAPE'], + [ + 'a non-boolean async', + { version: 1, captures: [[{ ...valid, async: 'no' }]] }, + 'CAPTURE_SHAPE', + ], + ['a fractional line', { version: 1, captures: [[{ ...valid, line: 1.5 }]] }, 'CAPTURE_SHAPE'], + ['line zero', { version: 1, captures: [[{ ...valid, line: 0 }]] }, 'CAPTURE_SHAPE'], + [ + 'a fractional column', + { version: 1, captures: [[{ ...valid, column: 1.5 }]] }, + 'CAPTURE_SHAPE', + ], + ['column zero', { version: 1, captures: [[{ ...valid, column: 0 }]] }, 'CAPTURE_SHAPE'], + [ + 'one bad frame among good ones', + { version: 1, captures: [[valid, { ...valid, line: 0 }]] }, + 'CAPTURE_SHAPE', + ], + ]; + const admitted = [ + ['64 frames', { version: 1, captures: [Array(64).fill(valid)] }], + [ + 'a native frame without a position', + { version: 1, captures: [[{ file: null, line: null, column: null, async: false }, valid]] }, + ], + ]; + // A stand-in run that imports the canary into its project and writes the given record. + const writing = (record) => async (_command, args, options) => { + const workflows = join(args.at(-1), '.relavium', 'workflows'); + mkdirSync(workflows, { recursive: true }); + writeFileSync( + join(workflows, `${options.env.RELAVIUM_FRAME_CAPTURE_CANARY}.relavium.yaml`), + '', + ); + if (record !== undefined) { + writeFileSync( + options.env.RELAVIUM_FRAME_CAPTURE_LOG, + typeof record === 'string' ? record : JSON.stringify(record), + ); + } + return { stdout: '', stderr: '' }; + }; + for (const [label, record, code] of cases) { + const work = fresh('record'); + mkdirSync(work, { mode: 0o700 }); + await assert.rejects( + captureFrames({ candidate: scratch, work, run: writing(record) }), + refuses(code), + label, + ); + } + for (const [label, record] of admitted) { + const work = fresh('record'); + mkdirSync(work, { mode: 0o700 }); + assert.deepEqual( + await captureFrames({ candidate: scratch, work, run: writing(record) }), + record.captures[0], + label, + ); + } }); test('the capture refuses a run that fails, never parses the canary or parses it twice', async () => { for (const [label, entry, code] of [ - ['a failed run', 'process.exit(3);', 'CAPTURE_FAILED'], - ['no canary parse', "import('yaml').then((y) => y.parse('a: 1'));", 'CAPTURE_COUNT'], + ['a failed run', `${IMPORTS}process.exit(3);\n`, 'CAPTURE_FAILED'], + [ + 'no canary parse', + `${IMPORTS}import('yaml').then((y) => y.parse('a: 1'));\n`, + 'CAPTURE_COUNT', + ], [ 'two canary parses', - "import { readFileSync } from 'node:fs'; import { parse } from 'yaml';" + - "const t = readFileSync(process.argv[3], 'utf8'); parse(t); parse(t);", + `${IMPORTS}import { readFileSync } from 'node:fs'; import { parse } from 'yaml';\n` + + "const t = readFileSync(process.argv[3], 'utf8'); parse(t); parse(t);\n", 'CAPTURE_COUNT', ], ]) { @@ -578,57 +791,176 @@ test('the capture refuses a run that fails, never parses the canary or parses it } }); -test('the preload refuses the keyring and leaves V8’s stack formatting as it found it', async () => { +/** Run `entry` as the finalized CLI under the preload, with only PATH and `env` set. */ +function preload(fixture, entry, env, args = []) { + writeFileSync(join(fixture.cli, 'dist', 'index.js'), entry); + return spawnSync( + process.execPath, + ['--import', pathToFileURL(PRELOAD).href, join(fixture.cli, 'dist', 'index.js'), ...args], + { encoding: 'utf8', timeout: 30_000, env: { PATH: process.env.PATH, ...env } }, + ); +} + +test('the preload refuses the keyring and native modules and leaves V8’s stack formatting alone', async () => { const fixture = await candidate(); const canary = 'frames-canary-preload'; const document = join(fixture.work, `${canary}.relavium.yaml`); - writeFileSync(document, `workflow:\n id: ${canary}\n`); - writeFileSync( - join(fixture.cli, 'dist', 'index.js'), + writeFileSync(document, `id: ${canary}\nv: yes\n`); + const log = join(fixture.work, 'frames.json'); + const configured = { + RELAVIUM_FRAME_CAPTURE_LOG: log, + RELAVIUM_FRAME_CAPTURE_PACKAGE: join(fixture.cli, 'package.json'), + RELAVIUM_FRAME_CAPTURE_CANARY: canary, + }; + const read = () => JSON.parse(readFileSync(log, 'utf8')); + const clear = () => rmSync(log, { force: true }); + let run = preload( + fixture, "import { readFileSync } from 'node:fs';\n" + - "import { Entry } from '@napi-rs/keyring';\n" + + "import { Entry, findCredentials } from '@napi-rs/keyring';\n" + "import { parse } from 'yaml';\n" + - 'let keyring;\n' + - "try { new Entry('relavium', 'probe'); keyring = 'native'; } catch (error) { keyring = error.message; }\n" + + 'const message = (action) => { try { action(); return "ran"; } catch (error) { return error.message; } };\n' + + "const keyring = [message(() => new Entry('relavium', 'probe')), message(() => findCredentials('relavium'))];\n" + + "const native = message(() => process.dlopen({ exports: {} }, '/nonexistent/addon.node'));\n" + "parse(readFileSync(process.argv[2], 'utf8'));\n" + // A formatter left installed would make `stack` an array of call sites, not a string. - 'process.stdout.write(JSON.stringify({ keyring, limit: Error.stackTraceLimit,' + - " stack: typeof new Error('x').stack }));\n", - ); - const run = spawnSync( - process.execPath, - [ - '--import', - new URL('frame-capture.mjs', import.meta.url).href, - join(fixture.cli, 'dist', 'index.js'), - document, - ], - { - encoding: 'utf8', - timeout: 30_000, - env: { - PATH: process.env.PATH, - RELAVIUM_FRAME_CAPTURE_LOG: join(fixture.work, 'frames.json'), - RELAVIUM_FRAME_CAPTURE_PACKAGE: join(fixture.cli, 'package.json'), - RELAVIUM_FRAME_CAPTURE_CANARY: canary, - }, - }, + "process.stdout.write(JSON.stringify({ keyring, native, limit: Error.stackTraceLimit, stack: typeof new Error('x').stack }));\n", + configured, + [document], ); assert.equal(run.status, 0, run.stderr); assert.deepEqual(JSON.parse(run.stdout), { - keyring: 'The symbolication run refuses the keyring', + keyring: [ + 'The symbolication run refuses the keyring', + 'The symbolication run refuses the keyring', + ], + native: 'The symbolication run refuses native modules', limit: 10, stack: 'string', }); - const record = JSON.parse(readFileSync(join(fixture.work, 'frames.json'), 'utf8')); - assert.equal(record.captures.length, 1); - // Owner-only, and never overwritten. - if (posixModes) assert.equal(statSync(join(fixture.work, 'frames.json')).mode & 0o777, 0o600); - // Without its configuration the preload refuses to run at all. - const bare = spawnSync( - process.execPath, - ['--import', new URL('frame-capture.mjs', import.meta.url).href, '--eval', ''], - { encoding: 'utf8', timeout: 30_000, env: { PATH: process.env.PATH } }, + assert.equal(read().captures.length, 1); + if (posixModes) assert.equal(statSync(log).mode & 0o777, 0o600); + // An explicit process.exit still writes the record. + clear(); + run = preload( + fixture, + "import { readFileSync } from 'node:fs'; import { parse } from 'yaml';\n" + + "parse(readFileSync(process.argv[2], 'utf8'));\nprocess.exit(0);\n", + configured, + [document], + ); + assert.equal(run.status, 0, run.stderr); + assert.equal(read().captures.length, 1); + // The first frame is the caller of parse, never the preload; parse options pass through. + clear(); + run = preload( + fixture, + "import { readFileSync } from 'node:fs'; import { parse } from 'yaml';\n" + + "const value = parse(readFileSync(process.argv[2], 'utf8'), { version: '1.1' });\n" + + 'process.stdout.write(JSON.stringify(value.v));\n', + configured, + [document], + ); + assert.equal(run.status, 0, run.stderr); + assert.equal(run.stdout, 'true'); + assert.equal( + read().captures[0][0].file, + pathToFileURL(join(fixture.cli, 'dist', 'index.js')).href, + ); + // An existing record is never overwritten. + writeFileSync(log, 'keep'); + preload(fixture, "import 'yaml';\n", configured); + assert.equal(readFileSync(log, 'utf8'), 'keep'); + // Asynchronous callers are marked async and a native frame has no file. + clear(); + run = preload( + fixture, + "import { readFileSync } from 'node:fs'; import { parse } from 'yaml';\n" + + 'async function inner(text) { await null; [text].map((t) => parse(t)); }\n' + + 'async function outer(text) { await inner(text); }\n' + + "await outer(readFileSync(process.argv[2], 'utf8'));\n", + configured, + [document], + ); + assert.equal(run.status, 0, run.stderr); + const frames = read().captures[0]; + assert.ok( + frames.some((frame) => frame.async === true), + 'an async frame', + ); + assert.ok( + frames.some((frame) => Object.hasOwn(frame, 'file') && frame.file === null), + 'a native frame', + ); + // An eval frame has no file name: it is recorded as null, never dropped. + clear(); + run = preload( + fixture, + "import { readFileSync } from 'node:fs'; import { parse } from 'yaml';\n" + + "const text = readFileSync(process.argv[2], 'utf8');\neval('parse(text)');\n", + configured, + [document], + ); + assert.equal(run.status, 0, run.stderr); + assert.ok(read().captures[0].every((frame) => Object.hasOwn(frame, 'file'))); + assert.ok(read().captures[0].some((frame) => frame.file === null)); + // A deep stack stops at the 64 frames the producer admits. + clear(); + run = preload( + fixture, + "import { readFileSync } from 'node:fs'; import { parse } from 'yaml';\n" + + "const text = readFileSync(process.argv[2], 'utf8');\n" + + 'function deep(n) { if (n === 0) return parse(text); return deep(n - 1); }\ndeep(100);\n', + configured, + [document], + ); + assert.equal(run.status, 0, run.stderr); + assert.equal(read().captures[0].length, 64); + // A platform keyring binding is refused by name. + clear(); + run = preload( + fixture, + "let result;\ntry { await import('@napi-rs/keyring-darwin-arm64'); result = 'loaded'; } catch (error) { result = error.message; }\nprocess.stdout.write(result);\n", + configured, ); - assert.notEqual(bare.status, 0); + assert.equal(run.stdout, 'Native keyring import refused'); + // Without a canary, a log or a package the preload refuses before the entry runs. + const marker = join(fixture.work, 'ran'); + for (const missing of Object.keys(configured)) { + const env = { ...configured }; + delete env[missing]; + clear(); + run = preload( + fixture, + `import { writeFileSync } from 'node:fs';\nwriteFileSync(${JSON.stringify(marker)}, 'x');\n`, + env, + ); + assert.notEqual(run.status, 0, missing); + assert.equal(existsSync(marker), false, missing); + assert.equal(existsSync(log), false, missing); + } +}); + +test('the keyring stand-in has every export the real package declares, and each refuses', () => { + // Read from the package's own declarations and entry, so the test loads no native binding. + const root = dirname(cliRequire.resolve('@napi-rs/keyring/package.json')); + const names = new Set([ + ...[ + ...readFileSync(join(root, 'index.d.ts'), 'utf8').matchAll( + /^export declare (?:class|function) (\w+)/gmu, + ), + ].map((match) => match[1]), + ...[ + ...readFileSync(join(root, 'index.js'), 'utf8').matchAll(/^module\.exports\.(\w+) =/gmu), + ].map((match) => match[1]), + ]); + assert.ok(names.size >= 4, 'the package declares its exports'); + for (const name of names) assert.equal(Object.hasOwn(keyringStub, name), true, name); + for (const [name, value] of Object.entries(keyringStub)) { + assert.throws( + () => (/^[A-Z]/u.test(name) ? new value() : value()), + /^Error: The symbolication run refuses the keyring$/u, + name, + ); + } }); From 990a5d1620a30d4206ddc4b3592e830fe52738fc Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sat, 24 Oct 2026 12:18:48 +0300 Subject: [PATCH 40/97] fix(repo): confine the symbolication run's module resolution Step 9a.2a review round 1 at fd6cd1cd was NONCLEAN with one confirmed medium from the security sitting: the capture child runs the candidate CLI from the producer's root in the shared temporary directory, so ws's optional require('bufferutil') resolved a module another local user could plant in /node_modules. It ran beside the private staging and the production still succeeded. - frame-capture.mjs registers a synchronous resolution hook (module.registerHooks, which sees require as well as import) that refuses any module resolving outside the candidate. The keyring stand-in moves into the same hook. A test plants such a module and shows Node loads it without the preload and refuses it, by require and by import, with it. The test fixture installs yaml inside the candidate in pnpm's layout. - The anchor ruling counts distinct call sites, so a repeated entry no longer counts twice. - The manifest admits exactly the package versions a receipt admits, through one isPackageVersion rule exported from binding.mjs. - Docs: the contract's 9a.2a section (module confinement, the Windows caveat of "owner-only", distinct call sites, the named limit), the tooling README, the register's round-1 record with the author's CI evidence at fd6cd1cd, and the re-check note on the triggered 9a.1 PLAT-9 deferral. Of thirteen mutants over the corrections, ten are killed and three are defensive. With bufferutil planted in TMPDIR the real production succeeds and the planted module never runs. Gates: lint:tools, typecheck:tools, prettier, test:acceptance-tools (882 of 882 under CI=true GITHUB_ACTIONS=true) and smoke:cli-package with the real producer and consumer. Refs: ADR-0111, #46 Co-Authored-By: Claude Opus 5.5 --- docs/reference/cli/packaging.md | 28 +++++++---- docs/roadmap/deferred-tasks.md | 3 +- .../phase-2.5.5-hardening-and-remediation.md | 24 +++++++++- tools/cli-package/README.md | 3 +- tools/cli-package/binding.mjs | 12 ++++- tools/cli-package/candidate-fixture.mjs | 17 ++++++- tools/cli-package/frame-capture-hooks.mjs | 40 ++++++++++++---- tools/cli-package/frame-capture.mjs | 18 +++++-- tools/cli-package/manifest.mjs | 4 +- tools/cli-package/manifest.test.mjs | 13 ++++- tools/cli-package/symbolicate.mjs | 10 +++- tools/cli-package/symbolicate.test.mjs | 47 ++++++++++++++++++- 12 files changed, 181 insertions(+), 38 deletions(-) diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index bb19d7f1..cc4b6ae4 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -347,6 +347,9 @@ refuses every artifact (the [Interim release guard ruling](../../roadmap/phases/ Step 9a.2a (a [recorded split](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24) of 9a.2, maintainer-approved on 2026-10-24) implements items 4 and 5 above for the package-test producer. The receipt stays `compiled_package_test` until 9a.2b's product map controls exist. +"Owner-only" in this section has the meaning the 9a.1 section gives it: mode 0700 or 0600 where +POSIX modes are observable; on Windows the private root inherits the temporary directory's ACL, +which is not verified. - **Strict maps.** The finalizer decodes every consumed upstream map's and every final map's `mappings` strictly — base64 VLQ only, no truncated or oversized value, segments of one, four or @@ -361,11 +364,12 @@ producer. The receipt stays `compiled_package_test` until 9a.2b's product map co staging (owner-only, never overwritten): version 1; the source commit, the commit's Git tree and the `dirty` flag; the package version; the build target; Node, pnpm, TypeScript, tsup and esbuild versions from the installed tools; SHA-256 digests of the lockfile, every package manifest and - every build configuration with the configurations it extends; every final JS file keyed as `dist/.js` with its SHA-256, its map's - package-relative path (`dist/.js.map`) and the staged map's SHA-256; a SHA-256 for every original source a final map names (repository-relative, no - content); and the tarball's SHA-256. The staged maps must still be the bytes the finalizer recorded - and each JS digest must be the tarball entry's. It holds no host path, credential or source text - and is removed with the staging. + every build configuration with the configurations it extends; every final JS file keyed as + `dist/.js` with its SHA-256, its map's package-relative path (`dist/.js.map`) and + the staged map's SHA-256; a SHA-256 for every original source a final map names + (repository-relative, no content); and the tarball's SHA-256. The staged maps must still be the + bytes the finalizer recorded and each JS digest must be the tarball entry's. It holds no host path, + credential or source text and is removed with the staging. - **Actual frames.** The seal then runs the finalized `dist` — which the inspection has proven byte-equal to the tarball entry by entry — under a tooling-only `node --import` preload while it imports a canary workflow (`relavium import`, with its home, project and captured frames in an @@ -376,13 +380,18 @@ producer. The receipt stays `compiled_package_test` until 9a.2b's product map co `yaml` package's `parse`, records V8's structured call sites for the canary document alone, restores the stack formatter and limit, replaces the keyring package with a module that refuses every use, and makes any native addon load refuse, so the run loads no native module and reads no - secret. Exactly one captured stack is required. + secret. Through a synchronous resolution hook that sees `require` as well as `import`, it also + refuses any module that resolves outside the candidate, so an optional dependency the candidate + does not install (ws's `bufferutil`, for example) can never be satisfied by a `node_modules` that + another local user creates above it in a shared temporary directory. Exactly one captured stack is + required. - **Symbolication.** Each frame inside the finalized `dist` must be a file the manifest and the tarball record (re-hashed after the run), its map must be the staged bytes the manifest records, and only a frame that lands exactly on a mapping segment counts. Five named call sites must be reached — three in the CLI and two in the inlined `@relavium/core`, each the one call to a named - function inside another, and the table itself refuses fewer than two core or three CLI call sites - — at the line and UTF-16 column that the matching source revision's syntax tree gives: for a + function inside another, and the table itself refuses fewer than two distinct core or three + distinct CLI call sites — at the line and UTF-16 column that the matching source revision's + syntax tree gives: for a clean candidate the checkout's Git objects at the recorded commit, read as the raw blob (no filter or text conversion) by Git without home, global or system configuration and without replace refs (the checkout's own configuration is still read), and for a dirty local one the candidate copy, @@ -393,7 +402,8 @@ producer. The receipt stays `compiled_package_test` until 9a.2b's product map co The capture child is not sandboxed: it runs the candidate CLI's runtime import closure as the producer's user, with network access, beside the private staging, confined only by its isolated -environment, its owner-only work directory and the refused keyring and native modules. `relavium +environment, its owner-only work directory, the refusal of any module outside the candidate and the +refused keyring and native modules. `relavium import` makes no network request, but nothing enforces that. This is a named limit of the package-test producer, owned by [deferred-tasks.md](../../roadmap/deferred-tasks.md#wave-3-step-9a2a--recorded-low-findings-2026-10-24). diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index 241d1b57..2fa984f4 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -598,7 +598,8 @@ real run that shows the shape. emptiness check (`mkdirSync` recursive plus `readdirSync`), so one seal can read the other's tarball. It needs user error: the local check always uses a fresh root. **Re-check:** require that the directory does not exist yet and create it without `recursive`, or take an exclusive lock file first. - *(nit · tools/cli-package/produce.mjs)* + *(nit · tools/cli-package/produce.mjs)* *Re-checked in Step 9a.2a (2026-10-24), which changed + `produce.mjs` only to add the symbolication work directory: unchanged, still needs user error.* - [ ] **The consumer's and the smoke worker's composition has no node test.** *Low, unchanged since Step 7b.* *(rigor pass, F6 and F7)* `consume.mjs` hands `completeConsumer` the `supportRoot` that `runInstalledSmoke` uses and gives the local source lookup `isolatedGitEnvironment()`, and the worker picks diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 2e5c98c9..063527f3 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -9722,5 +9722,25 @@ and of the corrections were then run against the corrected tests: 509 are killed changed and are replaced by new mutants, and the 41 survivors are equivalent or defensive (unreachable defaults, caches, a redundant clause, parser options, constants and the binary search's arithmetic). -**Remaining gates.** The full `pnpm run ci`, then one NEW complete cumulative round with no blocker, high or medium -finding and no regression, together with the separate scoped security sitting at the same head and bar. +**Author evidence at `fd6cd1cd`.** The full `pnpm run ci` (exit 0, about 70 minutes, with the real producer +and consumer smoke and every causal-control family) and `test:acceptance-tools` (881 of 881, plain and under +`CI=true GITHUB_ACTIONS=true`); on the clean tree, the package smoke and a direct production whose receipt +records `dirty: false` and whose anchors were read from the commit's Git objects. + +**Round 1 at `fd6cd1cd` — NONCLEAN.** Five lenses and a critic, every finding adversarially verified. The security +sitting reported one confirmed medium (SEC-9A2A-R1-1): the capture child ran the candidate CLI from the producer's +root in the shared temporary directory, so ws's optional `require('bufferutil')` resolved a module another local +user could plant in `/node_modules`, which ran beside the private staging while the production still +succeeded. Corrected: the preload registers a synchronous resolution hook (`module.registerHooks`, which sees +`require` as well as `import`) that refuses any module resolving outside the candidate; a test plants such a module +and shows that Node loads it without the preload and refuses it, by `require` and by `import`, with it. The test +fixture now installs `yaml` inside the candidate in pnpm's layout. Lows corrected in the same commit: the anchor +ruling counts distinct call sites (PLAN-9A2A-R1-1 = DMCS-9A2A-R1-1); the contract's 9a.2a section carries the +Windows caveat of "owner-only" (PLAN-9A2A-R1-2); the manifest admits exactly the package versions a receipt admits, +through one shared rule (CRIT-9A2A-R1-1); the triggered 9a.1 PLAT-9 deferral records its re-check (CRIT-9A2A-R1-2); +and this checkpoint records the author's CI evidence (CRIT-9A2A-R1-3). Of thirteen mutants over the corrections, ten +are killed and three are defensive: a prefix-sibling comparison that no resolution path reaches, a real-path call +on paths the resolver already returns real, and the hook API check on a Node that has the API. + +**Remaining gates.** Round 2 — one NEW complete cumulative round with no blocker, high or medium finding and no +regression, with the separate scoped security sitting at the same head and bar — then the push CI. diff --git a/tools/cli-package/README.md b/tools/cli-package/README.md index a1843f62..fbb73338 100644 --- a/tools/cli-package/README.md +++ b/tools/cli-package/README.md @@ -62,7 +62,8 @@ temporary directory that holds the isolated homes (`corepackHome` in `process.mj decodes every private map strictly (`source-map.mjs`). Before the receipt is written, the seal seals the private manifest (`manifest.mjs`) and symbolicates actual packed CLI and core frames, captured by a tooling-only preload while the CLI imports a canary into a private project -(`frame-capture.mjs`), through the strictly decoded maps of the chunks those frames land in against +(`frame-capture.mjs`, which refuses any module that resolves outside the candidate, the keyring and +native addons), through the strictly decoded maps of the chunks those frames land in against the matching source revision (`symbolicate.mjs`). The manifest is discarded with the staging, and the capture's home, project and frames with the private root. The receipt does **not** certify encrypted custody or a production release candidate (Steps 9b–9d), diff --git a/tools/cli-package/binding.mjs b/tools/cli-package/binding.mjs index d2236b4c..93713b8d 100644 --- a/tools/cli-package/binding.mjs +++ b/tools/cli-package/binding.mjs @@ -65,9 +65,19 @@ function sha(value) { function sourceSha(value) { return string(value, /^[0-9a-f]{40}$/u); } +const PACKAGE_VERSION = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/u; +/** + * Whether `value` is a package version a receipt admits: a SemVer core with an optional prerelease and + * build, at most 100 characters. The private manifest admits exactly the same versions. + * @param {unknown} value @returns {value is string} + */ +export function isPackageVersion(value) { + return typeof value === 'string' && value.length <= 100 && PACKAGE_VERSION.test(value); +} /** @param {unknown} value */ function version(value) { - return string(value, /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/u, 100); + if (!isPackageVersion(value)) refuse('VALUE'); + return value; } /** @param {unknown} value */ function nodeVersion(value) { diff --git a/tools/cli-package/candidate-fixture.mjs b/tools/cli-package/candidate-fixture.mjs index afa1cb43..27075164 100644 --- a/tools/cli-package/candidate-fixture.mjs +++ b/tools/cli-package/candidate-fixture.mjs @@ -10,6 +10,7 @@ import { Buffer } from 'node:buffer'; import { createHash } from 'node:crypto'; import { chmodSync, + cpSync, mkdirSync, mkdtempSync, readFileSync, @@ -30,7 +31,8 @@ import { MANIFEST_INPUTS, sealPrivateManifest } from './manifest.mjs'; export const cliRequire = createRequire(new URL('../../apps/cli/package.json', import.meta.url)); const { build } = createRequire(cliRequire.resolve('tsup/package.json'))('esbuild'); export const ts = cliRequire('typescript'); -const cliModules = fileURLToPath(new URL('../../apps/cli/node_modules', import.meta.url)); +/** The directory of an installed package the CLI resolves. */ +const installed = (name) => dirname(cliRequire.resolve(`${name}/package.json`)); export const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'relavium-candidate-fixture-'))); after(() => rmSync(scratch, { recursive: true, force: true })); export const digest = (bytes) => createHash('sha256').update(bytes).digest('hex'); @@ -152,7 +154,18 @@ export async function candidate({ sources = SOURCES, parent = scratch } = {}) { } put(root, 'apps/cli/tsup.config.ts', "export default {\n target: 'node22',\n};\n"); put(root, 'apps/cli/package.json', JSON.stringify({ name: 'relavium', type: 'module' })); - symlinkSync(cliModules, join(root, 'apps', 'cli', 'node_modules'), 'junction'); + // The CLI's installation: the build tools the test process reads link to the repository's, and + // `yaml`, the one runtime package the packed CLI loads, is a copy inside the candidate laid out as + // pnpm installs it (the package below the root's `node_modules/.pnpm`, linked from the CLI's), because + // the preload refuses any module outside the candidate. + const modules = join(root, 'apps', 'cli', 'node_modules'); + mkdirSync(join(modules, '@napi-rs'), { recursive: true }); + for (const name of ['typescript', 'tsup', '@napi-rs/keyring']) { + symlinkSync(installed(name), join(modules, ...name.split('/')), 'junction'); + } + const store = join(root, 'node_modules', '.pnpm', 'yaml', 'node_modules', 'yaml'); + cpSync(installed('yaml'), store, { recursive: true }); + symlinkSync(store, join(modules, 'yaml'), 'junction'); const cli = join(root, 'apps', 'cli'); const built = await build({ absWorkingDir: cli, diff --git a/tools/cli-package/frame-capture-hooks.mjs b/tools/cli-package/frame-capture-hooks.mjs index dfe1e796..79d86484 100644 --- a/tools/cli-package/frame-capture-hooks.mjs +++ b/tools/cli-package/frame-capture-hooks.mjs @@ -1,11 +1,31 @@ -// Loader hook of the symbolication preload (frame-capture.mjs): the packed CLI's keyring binding is -// replaced by a module that refuses every use, so the run loads no native module and reads no secret. -export async function resolve(specifier, context, nextResolve) { - if (specifier === '@napi-rs/keyring') - return { - url: new URL('./frame-capture-keyring.mjs', import.meta.url).href, - shortCircuit: true, - }; - if (specifier.startsWith('@napi-rs/keyring-')) throw new Error('Native keyring import refused'); - return nextResolve(specifier, context); +// Resolution hook of the symbolication preload (frame-capture.mjs), registered in-thread with +// `module.registerHooks` so that it sees `require` as well as `import`. The packed CLI's keyring +// binding is replaced by a module that refuses every use, so the run loads no native module and reads +// no secret. And no module may resolve outside the candidate: an optional `require` such as ws's +// `bufferutil` would otherwise fall through to a `node_modules` directory above the candidate, which +// another local user can create in a shared temporary directory. (The preload's own modules are +// loaded before the hook is registered, and the stand-in is returned without resolving.) +import { realpathSync } from 'node:fs'; +import { sep } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const KEYRING = new URL('./frame-capture-keyring.mjs', import.meta.url).href; + +/** + * A synchronous resolve hook confined to the candidate `root` (a real path). + * @param {string} root + */ +export function confinedResolve(root) { + return function resolve(specifier, context, nextResolve) { + if (specifier === '@napi-rs/keyring') return { url: KEYRING, shortCircuit: true }; + if (specifier.startsWith('@napi-rs/keyring-')) throw new Error('Native keyring import refused'); + const resolved = nextResolve(specifier, context); + if (resolved.url.startsWith('file:')) { + const path = realpathSync.native(fileURLToPath(resolved.url)); + if (path !== root && !path.startsWith(`${root}${sep}`)) { + throw new Error('The symbolication run refuses a module outside the candidate'); + } + } + return resolved; + }; } diff --git a/tools/cli-package/frame-capture.mjs b/tools/cli-package/frame-capture.mjs index 7ff0ca95..a1af2e86 100644 --- a/tools/cli-package/frame-capture.mjs +++ b/tools/cli-package/frame-capture.mjs @@ -5,18 +5,26 @@ * instance the CLI's import and require both reach — and records V8's structured call sites for the * canary document alone, so one synchronous stack crosses the CLI and the inlined `@relavium/core`. It * writes one owner-only record when the process exits; the producer refuses unless exactly one stack - * was captured. The keyring binding is replaced by a refusing module (frame-capture-hooks.mjs) and - * any native module load refuses. + * was captured. The keyring binding is replaced by a refusing module, no module may resolve outside + * the candidate (frame-capture-hooks.mjs), and any native module load refuses. */ -import { closeSync, openSync, writeSync } from 'node:fs'; -import { createRequire, register } from 'node:module'; +import { closeSync, openSync, realpathSync, writeSync } from 'node:fs'; +import { createRequire, registerHooks } from 'node:module'; +import { dirname, resolve } from 'node:path'; +import { confinedResolve } from './frame-capture-hooks.mjs'; const log = process.env.RELAVIUM_FRAME_CAPTURE_LOG; const manifest = process.env.RELAVIUM_FRAME_CAPTURE_PACKAGE; const canary = process.env.RELAVIUM_FRAME_CAPTURE_CANARY; if (!log || !manifest || !canary) throw new Error('The symbolication preload is not configured'); +if (typeof registerHooks !== 'function') { + throw new Error('The symbolication preload needs module.registerHooks'); +} -register(new URL('./frame-capture-hooks.mjs', import.meta.url)); +// The candidate root holds `apps/cli/package.json`; pnpm installs the CLI's dependencies below it. +registerHooks({ + resolve: confinedResolve(realpathSync.native(resolve(dirname(manifest), '..', '..'))), +}); // The import route needs no native module; any addon load refuses instead of running native code // beside the private staging. process.dlopen = () => { diff --git a/tools/cli-package/manifest.mjs b/tools/cli-package/manifest.mjs index 78237d63..0c56ec78 100644 --- a/tools/cli-package/manifest.mjs +++ b/tools/cli-package/manifest.mjs @@ -10,6 +10,7 @@ import { Buffer } from 'node:buffer'; import { createRequire } from 'node:module'; import { basename, join } from 'node:path'; import { isDeepStrictEqual } from 'node:util'; +import { isPackageVersion } from './binding.mjs'; import { ENGINE_PACKAGES, writePrivateFile } from './finalize.mjs'; import { readOrdinaryFile, sha256 } from './files.mjs'; import { isolatedGitEnvironment, runCommand } from './process.mjs'; @@ -117,8 +118,7 @@ export function validateManifest(value) { !GIT_OBJECT.test(String(source.commit)) || !GIT_OBJECT.test(String(source.tree)) || typeof source.dirty !== 'boolean' || - typeof value.packageVersion !== 'string' || - !VERSION.test(value.packageVersion) || + !isPackageVersion(value.packageVersion) || typeof value.target !== 'string' || !/^node\d{2,3}$/u.test(value.target) || !exact(toolchain, ['node', 'pnpm', 'typescript', 'tsup', 'esbuild']) || diff --git a/tools/cli-package/manifest.test.mjs b/tools/cli-package/manifest.test.mjs index f4b455ca..e08b8627 100644 --- a/tools/cli-package/manifest.test.mjs +++ b/tools/cli-package/manifest.test.mjs @@ -128,6 +128,12 @@ test('the package version and tarball digest are the inspected archive’s', asy const manifest = await seal(fixture); assert.equal(manifest.packageVersion, '9.8.7'); assert.equal(manifest.tarballSha256, 'e'.repeat(64)); + // Every version a receipt admits: a prerelease with build metadata, and exactly 100 characters. + for (const version of ['1.0.0-rc.1+build.5', `0.1.1-${'a'.repeat(94)}`]) { + const admitted = await candidate(); + admitted.archive.package.version = version; + assert.equal((await seal(admitted)).packageVersion, version); + } }); test('the tree is asked of the original checkout through isolated Git', async () => { @@ -272,7 +278,8 @@ test('the manifest refuses drifted maps and tarball entries, unpaired records an ], [ 'an uninstalled toolchain', - (fixture) => rmSync(join(fixture.root, 'apps', 'cli', 'node_modules')), + (fixture) => + rmSync(join(fixture.root, 'apps', 'cli', 'node_modules'), { recursive: true, force: true }), 'TOOLCHAIN', ], ]) { @@ -418,7 +425,9 @@ test('a sealed manifest that is not exactly the validated shape refuses on read' ['packageVersion as array', (m) => (m.packageVersion = ['0.1.1'])], ['packageVersion not semver', (m) => (m.packageVersion = '0.1')], ['packageVersion with trailing text', (m) => (m.packageVersion = '0.1.1 x')], - ['packageVersion oversized suffix', (m) => (m.packageVersion = `0.1.1-${'a'.repeat(65)}`)], + ['packageVersion over 100 characters', (m) => (m.packageVersion = `0.1.1-${'a'.repeat(95)}`)], + ['packageVersion with a plus in its build', (m) => (m.packageVersion = '0.1.1+a+b')], + ['packageVersion with an empty prerelease', (m) => (m.packageVersion = '0.1.1-')], ['target as array', (m) => (m.target = ['node22'])], ['target not nodeNN', (m) => (m.target = 'es2022')], ['toolchain extra key', (m) => (m.toolchain.extra = '1.0.0')], diff --git a/tools/cli-package/symbolicate.mjs b/tools/cli-package/symbolicate.mjs index ddf22c28..e4a055a0 100644 --- a/tools/cli-package/symbolicate.mjs +++ b/tools/cli-package/symbolicate.mjs @@ -279,8 +279,14 @@ export async function symbolicateCandidate({ revision = readSourceRevision, anchors = ANCHORS, }) { - // The maintainer's coverage ruling: at least two `@relavium/core` and three CLI call sites. - const within = (root) => anchors.filter((anchor) => anchor.source.startsWith(root)).length; + // The maintainer's coverage ruling: at least two `@relavium/core` and three CLI call sites, counted + // as distinct call sites, so a repeated entry does not count twice. + const within = (root) => + new Set( + anchors + .filter((anchor) => anchor.source.startsWith(root)) + .map((anchor) => JSON.stringify([anchor.source, anchor.within, anchor.call])), + ).size; if (within('packages/core/src/') < 2 || within('apps/cli/src/') < 3) refuse('ANCHORS'); const manifest = readPrivateManifest(staging); const frames = await capture({ candidate, work, run }); diff --git a/tools/cli-package/symbolicate.test.mjs b/tools/cli-package/symbolicate.test.mjs index 199c504d..dea098b2 100644 --- a/tools/cli-package/symbolicate.test.mjs +++ b/tools/cli-package/symbolicate.test.mjs @@ -172,6 +172,9 @@ test('the anchors keep the coverage ruling and resolve in the real repository', ANCHORS.slice(1), ANCHORS.slice(0, 4), ANCHORS.filter((anchor) => !anchor.source.startsWith('packages/core/')), + // A repeated entry is one call site. + [ANCHORS[0], ANCHORS[0], ANCHORS[2], ANCHORS[2], ANCHORS[2]], + [...ANCHORS.slice(1), ANCHORS[1]], // Call sites in other engine packages, or outside a package's sources, are not core anchors. [ ...cli, @@ -476,7 +479,7 @@ test('only synchronous frames inside the finalized dist count, and an unrecorded // resolution cache would still answer for one that already resolved it. const untooled = await candidate(); await seal(untooled); - rmSync(join(untooled.cli, 'node_modules')); + rmSync(join(untooled.cli, 'node_modules'), { recursive: true, force: true }); await assert.rejects( symbolicate(untooled, { capture: async () => frames }), refuses('TOOLCHAIN'), @@ -941,6 +944,48 @@ test('the preload refuses the keyring and native modules and leaves V8’s stack } }); +test('the preload refuses any module that resolves outside the candidate', async () => { + const fixture = await candidate(); + // A module another local user could plant in a `node_modules` above the candidate, as ws's optional + // `bufferutil` would find one in a shared temporary directory. + const name = `relavium-planted-${process.pid}`; + const planted = join(dirname(fixture.root), 'node_modules', name); + mkdirSync(planted, { recursive: true }); + const marker = join(fixture.work, 'planted-ran'); + writeFileSync(join(planted, 'package.json'), JSON.stringify({ name, main: 'index.js' })); + writeFileSync( + join(planted, 'index.js'), + `require('node:fs').writeFileSync(${JSON.stringify(marker)}, 'x');\n`, + ); + const entry = + "import { createRequire } from 'node:module';\n" + + `let required; try { createRequire(import.meta.url)(${JSON.stringify(name)}); required = 'loaded'; } catch (error) { required = error.message; }\n` + + `let imported; try { await import(${JSON.stringify(name)}); imported = 'loaded'; } catch (error) { imported = error.message; }\n` + + 'process.stdout.write(JSON.stringify({ required, imported }));\n'; + // Without the preload, Node's resolution walks up and runs it. + writeFileSync(join(fixture.cli, 'dist', 'index.js'), entry); + let run = spawnSync(process.execPath, [join(fixture.cli, 'dist', 'index.js')], { + encoding: 'utf8', + timeout: 30_000, + env: { PATH: process.env.PATH }, + }); + assert.equal(run.status, 0, run.stderr); + assert.deepEqual(JSON.parse(run.stdout), { required: 'loaded', imported: 'loaded' }); + assert.equal(existsSync(marker), true); + rmSync(marker); + // Under the preload both a require and an import refuse, and it never runs. + const canary = 'frames-canary-confined'; + run = preload(fixture, entry, { + RELAVIUM_FRAME_CAPTURE_LOG: join(fixture.work, 'frames.json'), + RELAVIUM_FRAME_CAPTURE_PACKAGE: join(fixture.cli, 'package.json'), + RELAVIUM_FRAME_CAPTURE_CANARY: canary, + }); + assert.equal(run.status, 0, run.stderr); + const refusal = 'The symbolication run refuses a module outside the candidate'; + assert.deepEqual(JSON.parse(run.stdout), { required: refusal, imported: refusal }); + assert.equal(existsSync(marker), false); +}); + test('the keyring stand-in has every export the real package declares, and each refuses', () => { // Read from the package's own declarations and entry, so the test loads no native binding. const root = dirname(cliRequire.resolve('@napi-rs/keyring/package.json')); From a92a76023b29e5cb6b541049a08186d60688e700 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sat, 24 Oct 2026 13:11:24 +0300 Subject: [PATCH 41/97] fix(repo): refuse a producer Node without module.registerHooks Step 9a.2a review round 2 at 990a5d16 reported no blocker, high or medium finding. Three lenses, however, found a regression the round-1 correction introduced: module.registerHooks exists from Node 22.15, the workspace admits Node 22.13, and on 22.13 or 22.14 every local production refused with the generic CAPTURE_FAILED, because the preload's named import failed at link time and its guard never ran. The amended bar requires no regression, so 9a.2a is not accepted at round 2. - produce.mjs refuses an unsupported producer Node with PACKED_CLI_PRODUCER_NODE before anything exists (captureSupported in symbolicate.mjs), and tests cover both a port and the default. - frame-capture.mjs reads registerHooks from the module object, so its own guard runs on an older Node; a test removes the API and sees it. - The contract and the tooling README name the Node 22.15 minimum. - The security sitting's low is corrected as wording: the keyring, native-addon and module refusals hold in the capture process itself, not in a thread or subprocess the candidate starts without the preload. The contract, the preload's docblock and the SEC-9A2A-2 record say so, and the record notes its re-check. - The register records round 2. All five mutants over the change are killed. Gates: lint:tools, typecheck:tools, prettier, test:acceptance-tools (884 of 884 under CI=true GITHUB_ACTIONS=true) and smoke:cli-package with the real producer and consumer. Refs: ADR-0111, #46 Co-Authored-By: Claude Opus 5.5 --- docs/reference/cli/packaging.md | 14 ++++--- docs/roadmap/deferred-tasks.md | 13 ++++-- .../phase-2.5.5-hardening-and-remediation.md | 18 ++++++++- tools/cli-package/README.md | 3 +- tools/cli-package/frame-capture.mjs | 9 +++-- tools/cli-package/produce.mjs | 5 +++ tools/cli-package/produce.test.mjs | 40 +++++++++++++++++++ tools/cli-package/symbolicate.mjs | 11 ++++- tools/cli-package/symbolicate.test.mjs | 39 ++++++++++++++++++ 9 files changed, 135 insertions(+), 17 deletions(-) diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index cc4b6ae4..b9bf6f8d 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -349,7 +349,9 @@ of 9a.2, maintainer-approved on 2026-10-24) implements items 4 and 5 above for t producer. The receipt stays `compiled_package_test` until 9a.2b's product map controls exist. "Owner-only" in this section has the meaning the 9a.1 section gives it: mode 0700 or 0600 where POSIX modes are observable; on Windows the private root inherits the temporary directory's ACL, -which is not verified. +which is not verified. The producer needs Node 22.15 or later, whose `module.registerHooks` the +preload uses (CI's pack jobs run the `.nvmrc` major); on an older Node it refuses with +`PACKED_CLI_PRODUCER_NODE` before anything is built. - **Strict maps.** The finalizer decodes every consumed upstream map's and every final map's `mappings` strictly — base64 VLQ only, no truncated or oversized value, segments of one, four or @@ -383,8 +385,10 @@ which is not verified. secret. Through a synchronous resolution hook that sees `require` as well as `import`, it also refuses any module that resolves outside the candidate, so an optional dependency the candidate does not install (ws's `bufferutil`, for example) can never be satisfied by a `node_modules` that - another local user creates above it in a shared temporary directory. Exactly one captured stack is - required. + another local user creates above it in a shared temporary directory. These refusals are installed + in the capture process itself: a thread or subprocess the candidate starts without the preload (an + eval Worker or a `node -e` child, for example) is not covered, and `relavium import` starts none, + though nothing enforces that. Exactly one captured stack is required. - **Symbolication.** Each frame inside the finalized `dist` must be a file the manifest and the tarball record (re-hashed after the run), its map must be the staged bytes the manifest records, and only a frame that lands exactly on a mapping segment counts. Five named call sites must be @@ -402,8 +406,8 @@ which is not verified. The capture child is not sandboxed: it runs the candidate CLI's runtime import closure as the producer's user, with network access, beside the private staging, confined only by its isolated -environment, its owner-only work directory, the refusal of any module outside the candidate and the -refused keyring and native modules. `relavium +environment, its owner-only work directory and, within the capture process only, the refusal of any +module outside the candidate and the refused keyring and native modules. `relavium import` makes no network request, but nothing enforces that. This is a named limit of the package-test producer, owned by [deferred-tasks.md](../../roadmap/deferred-tasks.md#wave-3-step-9a2a--recorded-low-findings-2026-10-24). diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index 2fa984f4..119cd728 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -680,10 +680,15 @@ that shows the shape. - [ ] **The symbolication child is not sandboxed.** *Low; a named limit.* *(rigor pass, SEC-9A2A-2)* `captureFrames` runs the candidate CLI's runtime import closure — packages that before 9a.2a ran only in the consumer jobs — as the producer's user, with network access, in the job whose sibling `staging` directory - holds the private maps and manifest. Its isolated environment, owner-only work directory and the refused - keyring and native addons confine it; nothing enforces that `relavium import` stays offline. packaging.md - names the limit. **Re-check:** when Step 9b designs the custody job, run the capture child under an OS - sandbox or a separate user that cannot read `staging`. *(low · tools/cli-package/symbolicate.mjs)* + holds the private maps and manifest. Its isolated environment and owner-only work directory confine it, + and, within the capture process only, the refusal of any module outside the candidate and the refused + keyring and native addons (added after review round 1); a thread or subprocess the candidate started + without the preload (an eval Worker, a `node -e` child) would have none of them, and `relavium import` + starts none, though nothing enforces that or that it stays offline. packaging.md names the limit. + **Re-check:** when Step 9b designs the custody job, run the capture child and every descendant under an + OS sandbox or a separate user that cannot read `staging`. *(low · tools/cli-package/symbolicate.mjs)* + *Re-checked after rounds 1 and 2 (2026-10-24): the module confinement narrows the limit; the record and + packaging.md now say the refusals are in-process only.* - [ ] **Two packs of one source can differ in the packed manifest's key order.** *Low, for Step 9c.* *(rigor pass, integration lens)* Two productions of the same tree packed the five `@relavium/*` `devDependencies`, which pnpm rewrites from `workspace:*` to versions at pack time, in different orders (`core, llm, shared, diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 063527f3..36517a26 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -9742,5 +9742,19 @@ and this checkpoint records the author's CI evidence (CRIT-9A2A-R1-3). Of thirte are killed and three are defensive: a prefix-sibling comparison that no resolution path reaches, a real-path call on paths the resolver already returns real, and the hook API check on a Node that has the API. -**Remaining gates.** Round 2 — one NEW complete cumulative round with no blocker, high or medium finding and no -regression, with the separate scoped security sitting at the same head and bar — then the push CI. +**Round 2 at `990a5d16` — not accepted.** The plan lens was clean; the round reported no blocker, high or medium +finding, and the security sitting one low. But three lenses independently found a regression the round-1 +correction introduced (DMCS-9A2A-R2-1 = TCA-9A2A-R2-1 = INT-9A2A-R2-1, rated low): `module.registerHooks` exists +from Node 22.15, the workspace admits Node 22.13, and on 22.13 or 22.14 every local production now refused with the +generic `CAPTURE_FAILED` code, because the preload's named import failed at link time and its guard never ran. +Since the amended bar requires no regression, the increment is not accepted at round 2. Corrected: the producer +refuses an unsupported Node with `PACKED_CLI_PRODUCER_NODE` before anything exists (`captureSupported`), the +preload reads the API from the module object so its own guard runs, and the contract and README name the minimum. +The security sitting's low (SEC-9A2A-R2-1) is corrected as wording: the keyring, native-addon and module refusals +hold in the capture process itself, not in a thread or subprocess the candidate starts without the preload; the +contract, the preload's docblock and the SEC-9A2A-2 record say so, and that record notes its re-check +(CRIT-9A2A-R2-1). + +**Remaining gates.** Round 3 — one NEW complete cumulative round with no blocker, high or medium finding and no +regression, with the separate scoped security sitting at the same head and bar — then the full `pnpm run ci` and +the push CI. diff --git a/tools/cli-package/README.md b/tools/cli-package/README.md index fbb73338..49157f4f 100644 --- a/tools/cli-package/README.md +++ b/tools/cli-package/README.md @@ -63,7 +63,8 @@ decodes every private map strictly (`source-map.mjs`). Before the receipt is wri the private manifest (`manifest.mjs`) and symbolicates actual packed CLI and core frames, captured by a tooling-only preload while the CLI imports a canary into a private project (`frame-capture.mjs`, which refuses any module that resolves outside the candidate, the keyring and -native addons), through the strictly decoded maps of the chunks those frames land in against +native addons in its own process, and needs Node 22.15 or later; an older producer Node refuses with +`PACKED_CLI_PRODUCER_NODE` before building), through the strictly decoded maps of the chunks those frames land in against the matching source revision (`symbolicate.mjs`). The manifest is discarded with the staging, and the capture's home, project and frames with the private root. The receipt does **not** certify encrypted custody or a production release candidate (Steps 9b–9d), diff --git a/tools/cli-package/frame-capture.mjs b/tools/cli-package/frame-capture.mjs index a1af2e86..6fb817e3 100644 --- a/tools/cli-package/frame-capture.mjs +++ b/tools/cli-package/frame-capture.mjs @@ -6,10 +6,11 @@ * canary document alone, so one synchronous stack crosses the CLI and the inlined `@relavium/core`. It * writes one owner-only record when the process exits; the producer refuses unless exactly one stack * was captured. The keyring binding is replaced by a refusing module, no module may resolve outside - * the candidate (frame-capture-hooks.mjs), and any native module load refuses. + * the candidate (frame-capture-hooks.mjs), and any native module load refuses. These hold in this + * process: a thread or subprocess the candidate starts without the preload is not covered. */ import { closeSync, openSync, realpathSync, writeSync } from 'node:fs'; -import { createRequire, registerHooks } from 'node:module'; +import nodeModule, { createRequire } from 'node:module'; import { dirname, resolve } from 'node:path'; import { confinedResolve } from './frame-capture-hooks.mjs'; @@ -17,12 +18,12 @@ const log = process.env.RELAVIUM_FRAME_CAPTURE_LOG; const manifest = process.env.RELAVIUM_FRAME_CAPTURE_PACKAGE; const canary = process.env.RELAVIUM_FRAME_CAPTURE_CANARY; if (!log || !manifest || !canary) throw new Error('The symbolication preload is not configured'); -if (typeof registerHooks !== 'function') { +if (typeof nodeModule.registerHooks !== 'function') { throw new Error('The symbolication preload needs module.registerHooks'); } // The candidate root holds `apps/cli/package.json`; pnpm installs the CLI's dependencies below it. -registerHooks({ +nodeModule.registerHooks({ resolve: confinedResolve(realpathSync.native(resolve(dirname(manifest), '..', '..'))), }); // The import route needs no native module; any addon load refuses instead of running native code diff --git a/tools/cli-package/produce.mjs b/tools/cli-package/produce.mjs index 20c81642..3a625d0c 100644 --- a/tools/cli-package/produce.mjs +++ b/tools/cli-package/produce.mjs @@ -24,6 +24,7 @@ import { assertNoBuildCache, globalExcludesFile, prepareCandidate } from './cand import { within } from './paths.mjs'; import { exitOnInterrupt, isolatedEnvironment, runPnpm } from './process.mjs'; import { sealPackage } from './seal.mjs'; +import { captureSupported } from './symbolicate.mjs'; import { emitFreshPackage } from './pipeline.mjs'; /** @@ -42,6 +43,7 @@ import { emitFreshPackage } from './pipeline.mjs'; * @param {{repository: string, destination: string, environment: Record, * temporary?: string, ports?: {prepare?: typeof prepareCandidate, emit?: typeof emitFreshPackage, * seal?: typeof sealPackage, noBuildCache?: typeof assertNoBuildCache, + * capturable?: typeof captureSupported, * excludesFile?: typeof globalExcludesFile, pnpm?: typeof runPnpm, * remove?: (path: string) => void}}} options */ @@ -57,11 +59,14 @@ export async function producePackage({ emit = emitFreshPackage, seal = sealPackage, noBuildCache = assertNoBuildCache, + capturable = captureSupported, excludesFile = globalExcludesFile, pnpm = runPnpm, // Retries absorb a child killed by the interrupt that was still finishing a write. remove = (path) => rmSync(path, { recursive: true, force: true, maxRetries: 3 }), } = ports; + // The seal's symbolication needs Node 22.15 or later; refuse an older producer before anything exists. + assert.ok(capturable(), 'PACKED_CLI_PRODUCER_NODE'); mkdirSync(destination, { recursive: true }); assert.deepEqual(readdirSync(destination), [], 'PACKED_CLI_FRESH_DESTINATION_REQUIRED'); const home = realpathSync.native(mkdtempSync(join(temporary, 'relavium-package-build-'))); diff --git a/tools/cli-package/produce.test.mjs b/tools/cli-package/produce.test.mjs index 933d73b4..325b7dfc 100644 --- a/tools/cli-package/produce.test.mjs +++ b/tools/cli-package/produce.test.mjs @@ -12,6 +12,7 @@ import { symlinkSync, writeFileSync, } from 'node:fs'; +import nodeModule from 'node:module'; import { tmpdir } from 'node:os'; import { basename, dirname, join, sep } from 'node:path'; import { after, test } from 'node:test'; @@ -216,6 +217,45 @@ test('another pnpm than the pinned one refuses before anything is prepared', asy assert.deepEqual(leftovers(temporary), []); }); +test('a producer Node that cannot run the capture refuses before anything exists', async () => { + const temporary = fresh('tmp'); + const destination = join(fresh('artifact-parent'), 'artifact'); + const { seen, ports: fakes } = ports(); + fakes.capturable = () => false; + await assert.rejects( + producePackage({ + repository: fresh('original-checkout'), + destination, + environment: {}, + temporary, + ports: fakes, + }), + { message: 'PACKED_CLI_PRODUCER_NODE' }, + ); + assert.equal(seen.prepare, undefined); + assert.equal(existsSync(destination), false); + assert.deepEqual(leftovers(temporary), []); + // By default the producer asks this Node itself. + const registerHooks = nodeModule.registerHooks; + delete nodeModule.registerHooks; + try { + const { ports: defaults } = ports(); + delete defaults.capturable; + await assert.rejects( + producePackage({ + repository: fresh('original-checkout'), + destination: join(fresh('artifact-parent'), 'artifact'), + environment: {}, + temporary, + ports: defaults, + }), + { message: 'PACKED_CLI_PRODUCER_NODE' }, + ); + } finally { + nodeModule.registerHooks = registerHooks; + } +}); + test( 'the private root and staging are exactly 0700 under a umask without the owner write bit', { skip: !posixModes }, diff --git a/tools/cli-package/symbolicate.mjs b/tools/cli-package/symbolicate.mjs index e4a055a0..e61fedbe 100644 --- a/tools/cli-package/symbolicate.mjs +++ b/tools/cli-package/symbolicate.mjs @@ -13,7 +13,7 @@ import { Buffer } from 'node:buffer'; import { randomBytes } from 'node:crypto'; import { lstatSync, mkdirSync, realpathSync, writeFileSync } from 'node:fs'; -import { createRequire } from 'node:module'; +import nodeModule, { createRequire } from 'node:module'; import { basename, dirname, join } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { readPrivateManifest } from './manifest.mjs'; @@ -57,6 +57,15 @@ export const ANCHORS = Object.freeze([ ]); const PRELOAD = fileURLToPath(new URL('./frame-capture.mjs', import.meta.url)); + +/** + * Whether this Node can run the capture: the preload confines module resolution through + * `module.registerHooks` (Node 22.15 and later), which the workspace's Node 22.13 floor lacks. + * @param {{registerHooks?: unknown}} [module] + */ +export function captureSupported(module = nodeModule) { + return typeof module.registerHooks === 'function'; +} const FRAME_LIMIT = 64; /** A workflow the CLI accepts, whose id carries the canary the preload watches for. */ diff --git a/tools/cli-package/symbolicate.test.mjs b/tools/cli-package/symbolicate.test.mjs index dea098b2..eef63cad 100644 --- a/tools/cli-package/symbolicate.test.mjs +++ b/tools/cli-package/symbolicate.test.mjs @@ -38,6 +38,7 @@ import { SymbolicationError, anchorPosition, captureFrames, + captureSupported, readSourceRevision, symbolicateCandidate, } from './symbolicate.mjs'; @@ -986,6 +987,44 @@ test('the preload refuses any module that resolves outside the candidate', async assert.equal(existsSync(marker), false); }); +test('the capture needs module.registerHooks, and the preload refuses without it', async () => { + assert.equal(captureSupported(), true); + assert.equal(captureSupported({}), false); + assert.equal(captureSupported({ registerHooks: () => {} }), true); + // On a Node without the API the preload's own guard runs: it refuses before the entry runs. + const fixture = await candidate(); + const shim = join(fixture.work, 'no-register-hooks.cjs'); + writeFileSync(shim, "delete require('node:module').registerHooks;\n"); + const marker = join(fixture.work, 'ran'); + writeFileSync( + join(fixture.cli, 'dist', 'index.js'), + `import { writeFileSync } from 'node:fs';\nwriteFileSync(${JSON.stringify(marker)}, 'x');\n`, + ); + const run = spawnSync( + process.execPath, + [ + '--require', + shim, + '--import', + pathToFileURL(PRELOAD).href, + join(fixture.cli, 'dist', 'index.js'), + ], + { + encoding: 'utf8', + timeout: 30_000, + env: { + PATH: process.env.PATH, + RELAVIUM_FRAME_CAPTURE_LOG: join(fixture.work, 'frames.json'), + RELAVIUM_FRAME_CAPTURE_PACKAGE: join(fixture.cli, 'package.json'), + RELAVIUM_FRAME_CAPTURE_CANARY: 'frames-canary-old-node', + }, + }, + ); + assert.notEqual(run.status, 0); + assert.match(run.stderr, /The symbolication preload needs module\.registerHooks/u); + assert.equal(existsSync(marker), false); +}); + test('the keyring stand-in has every export the real package declares, and each refuses', () => { // Read from the package's own declarations and entry, so the test loads no native binding. const root = dirname(cliRequire.resolve('@napi-rs/keyring/package.json')); From f65999a6d3eeeaf12afbaa6904a386b38b6bad2f Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sat, 24 Oct 2026 14:03:37 +0300 Subject: [PATCH 42/97] docs(repo): record the Step 9a.2a acceptance Step 9a.2a is independently accepted on development at a92a7602 under the 2026-10-23 bar. Round 3, a new complete cumulative review of 6277e375..a92a7602, reported no blocker, high or medium finding and no regression; its separate security sitting and the plan, correctness and integration lenses and the critic were clean. Rounds 1 and 2 were not accepted: round 1 found a medium (module resolution outside the candidate), and round 2 a regression its correction introduced (the producer's Node minimum). - The register's 9a.2a row, the checkpoint and a new acceptance section. - deferred-tasks.md records round 3's low (symbolication's read bounds have no test) and one test nit (an injected anchor table), and the PLAT-9 re-check note names both produce.mjs changes. - Status lines in CLAUDE.md, AGENTS.md, current.md and the CLI reference. This commit changes no code. Refs: ADR-0111, #46 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 ++-- CLAUDE.md | 4 ++-- docs/reference/cli/README.md | 2 +- docs/reference/cli/packaging.md | 2 +- docs/roadmap/current.md | 6 ++--- docs/roadmap/deferred-tasks.md | 24 +++++++++++++++++-- .../phase-2.5.5-hardening-and-remediation.md | 21 ++++++++++++---- 7 files changed, 48 insertions(+), 15 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f2a23147..1d4923d9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,7 +117,7 @@ explicit; source CI does not certify later documentation commits. The joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now -accepted on `development`, below; Step 9a.2a is implemented and under review; Steps 9a.2b–19 have not started) and whole-wave acceptance remains Step 19. +accepted on `development`, below, as is Step 9a.2a; Steps 9a.2b–19 have not started) and whole-wave acceptance remains Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification @@ -130,7 +130,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the private manifest and symbolication) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-implementation-checkpoint--2026-10-24)). `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b is next. `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. See [docs/roadmap/current.md](docs/roadmap/current.md) for live status. diff --git a/CLAUDE.md b/CLAUDE.md index 13699342..f088e339 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -206,7 +206,7 @@ explicit; source CI does not certify later documentation commits. The joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now -accepted on `development`, below; Step 9a.2a is implemented and under review; Steps 9a.2b–19 have not started) and whole-wave acceptance remains Step 19. +accepted on `development`, below, as is Step 9a.2a; Steps 9a.2b–19 have not started) and whole-wave acceptance remains Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification @@ -219,7 +219,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the private manifest and symbolication) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-implementation-checkpoint--2026-10-24)). `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b is next. `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. For live status, per-PR history, milestone dates, and open obligations, see the canonical diff --git a/docs/reference/cli/README.md b/docs/reference/cli/README.md index ec5c5e82..751cc665 100644 --- a/docs/reference/cli/README.md +++ b/docs/reference/cli/README.md @@ -13,5 +13,5 @@ Part of [reference/](../README.md). | [commands.md](commands.md) | The full `relavium` command surface (run, chat, chat-resume/list/export, agent run, list, create, import, export, logs, gate, init, provider) plus interactive TUI vs CI JSON mode. | | [home.md](home.md) | The bare-invocation interactive **Home** — the TTY gate, the read-only management strip over `history.db`, the Home↔chat mode machine + keymap, bracketed paste, the min-terminal degrade, and the signal/exit-code lifecycle (2.5.B / ADR-0054). | | [chat-session.md](chat-session.md) | The `relavium chat` agent-session REPL — entry, agent/model selection, the multi-turn loop, streaming, tool availability, the `--json` session-event stream, and exit code 4. | -| [packaging.md](packaging.md) | Wave 3 finite runtime graph, installed-package consumers and Step 8 native activation accepted on `development`; the Step 9a.1 candidate build and finalizer are accepted on `development` and Step 9a.2a's private manifest and symbolication are implemented (acceptance pending); the remaining Step 9 map/custody and release gates remain staged. | +| [packaging.md](packaging.md) | Wave 3 finite runtime graph, installed-package consumers and Step 8 native activation accepted on `development`; the Step 9a.1 candidate build and finalizer are accepted on `development` and Step 9a.2a's private manifest and symbolication are accepted on `development`; the remaining Step 9 map/custody and release gates remain staged. | | [agent-run-fixture.md](agent-run-fixture.md) | The `relavium agent run --fixture` cassette format — a `StreamChunk[][]` JSON recording of an agent run's LLM stream, for deterministic offline replay. | diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index b9bf6f8d..5473f483 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -1,6 +1,6 @@ # CLI Packaging Contract -- **Status**: Compiled graph, installed package and Step 8 native activation independently accepted on `development`; the Step 9a.1 candidate build and finalizer are independently accepted on `development` (2026-10-24); Step 9a.2a's private manifest and symbolication are implemented, with independent acceptance pending; the remaining Step 9 map/custody and release gates remain staged +- **Status**: Compiled graph, installed package and Step 8 native activation independently accepted on `development`; the Step 9a.1 candidate build and finalizer are independently accepted on `development` (2026-10-24); Step 9a.2a's strict decoder, private manifest and symbolication are independently accepted on `development` (2026-10-24); the remaining Step 9 map/custody and release gates remain staged - **Date**: 2026-10-15 - **Related**: [ADR-0111](../../decisions/0111-cli-meta-operations-use-a-lazy-packaged-runtime.md), [commands.md](commands.md), [release runbook](../../runbooks/release-a-surface.md) diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index f1454965..3e765762 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -138,7 +138,7 @@ The [deferred register](deferred-tasks.md) retains its existing limits; closure **Wave 3 Steps 1–8 merged to `main` on 2026-10-18 as [PR #95](https://github.com/HodeTech/Relavium/pull/95), merge commit `6d2a7372` (22:44 UTC). Wave 3 itself is not complete: Steps 9–19 had not started at -that merge (Step 9a.1 is now accepted on `development` and Step 9a.2a is implemented and under review; Steps 9a.2b–19 have not started) and whole-wave acceptance +that merge (Steps 9a.1 and 9a.2a are now accepted on `development`; Steps 9a.2b–19 have not started) and whole-wave acceptance remains Step 19.** The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) joins the completed scope, every review attribution, the independent review's dispositions and the publication state observed at merge. It was the maintainer-authorized, scoped publication checkpoint. @@ -172,7 +172,7 @@ bar and round cap, as its medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, by the author alone). Its other findings are recorded in -[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is independently accepted on `development` at `cf97b7f1` (2026-10-24): its round 13 reported no blocker, high or medium finding and its security sitting was clean, and the round's low findings are recorded in deferred-tasks.md ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)); Step 9a.2 is split into 9a.2a and 9a.2b, and Step 9a.2a (the private manifest and symbolication) is implemented on `development` and under review ([checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-implementation-checkpoint--2026-10-24)); the substrate's +[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is independently accepted on `development` at `cf97b7f1` (2026-10-24): its round 13 reported no blocker, high or medium finding and its security sitting was clean, and the round's low findings are recorded in deferred-tasks.md ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)); Step 9a.2 is split into 9a.2a and 9a.2b, and Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24) after three rounds, the third with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)); Step 9a.2b is next; the substrate's 9a.1–9d increments carry a dedicated archive/crypto security sitting. Steps 10–19 follow the [increment table](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) and end with whole-wave acceptance at Step 19. `development` is pushed after each accepted increment; PR #95 no longer bounds the @@ -1141,7 +1141,7 @@ an ESLint selector rather than a list. ### Wave 3 — Clear the ground -**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Step 9a.1 accepted on `development` (2026-10-24), Step 9a.2a under review, Steps 9a.2b–19 NOT STARTED — 2026-10-24.** +**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Steps 9a.1 and 9a.2a accepted on `development` (2026-10-24), Steps 9a.2b–19 NOT STARTED — 2026-10-24.** ADR-0110–0113, their linked contracts and the nineteen-increment plan were approved on 2026-10-16. [Steps 1–8 were independently accepted](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) on `development`, with whole Step 8 reviewed at `c83c8575`, and published as the scoped checkpoint diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index 119cd728..cf4559d7 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -2,7 +2,7 @@ > Status: Living -> Last updated: 2026-10-24 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Step 9a.1 is independently accepted on `development` with its recorded low findings below, Step 9a.2a is implemented and under review with its recorded low findings below, Steps 9a.2b–19 not started; the PR95 increment is closed on `development` under the maintainer's 2026-10-21 round cap). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's edge cases with the maintainer's 2026-10-21 acceptance bar and round cap and the 2026-10-22 rulings. Existing approved deferrals remain. +> Last updated: 2026-10-24 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Step 9a.1 is independently accepted on `development` with its recorded low findings below, Step 9a.2a is independently accepted on `development` with its recorded low findings below, Steps 9a.2b–19 not started; the PR95 increment is closed on `development` under the maintainer's 2026-10-21 round cap). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's edge cases with the maintainer's 2026-10-21 acceptance bar and round cap and the 2026-10-22 rulings. Existing approved deferrals remain. > The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) records completed prerequisites; full M/N lifecycle and other scheduled work remain with their owners. > > Earlier triage: the Phase-2.6 rewrite triaged every open item, the 2026-07-19 full-project @@ -599,7 +599,8 @@ real run that shows the shape. needs user error: the local check always uses a fresh root. **Re-check:** require that the directory does not exist yet and create it without `recursive`, or take an exclusive lock file first. *(nit · tools/cli-package/produce.mjs)* *Re-checked in Step 9a.2a (2026-10-24), which changed - `produce.mjs` only to add the symbolication work directory: unchanged, still needs user error.* + `produce.mjs` to add the symbolication work directory and the producer-Node refusal, which runs before the + emptiness check: unchanged, still needs user error.* - [ ] **The consumer's and the smoke worker's composition has no node test.** *Low, unchanged since Step 7b.* *(rigor pass, F6 and F7)* `consume.mjs` hands `completeConsumer` the `supportRoot` that `runInstalledSmoke` uses and gives the local source lookup `isolatedGitEnvironment()`, and the worker picks @@ -699,6 +700,25 @@ that shows the shape. never compare a rebuilt digest, or make the published manifest's key order deterministic, with a test that packs twice and compares the digests. *(low · tools/cli-package/prepack.mjs and pnpm's pack step)* +Round 3, the acceptance round at `a92a7602` (no blocker, high or medium finding and no regression; the security +sitting, plan, correctness and integration lenses and the critic clean), reported three verified findings. The +wording nit is corrected in the acceptance commit, which changes no code (the PLAT-9 re-check note above now +names both `produce.mjs` changes, INT-9A2A-R3-1); the two test gaps are recorded here. + +- [ ] **Symbolication's three read bounds have no test.** *Low.* *(round 3, TCA-9A2A-R3-1)* `symbolicate.mjs` + reads the dirty source copy under 8 MiB (`readSourceRevision`), the staged map under 16 MiB (`mapsOf`) and + each re-hashed frame file under 16 MiB; raising all three tenfold leaves `symbolicate.test.mjs` passing. It is + defense in depth: the manifest step reads the same maps and sources under the same bounds, and every file is + bound by digest. **Re-check:** one case per site, as `manifest.test.mjs`'s bounded reads do — pad the dirty + copy, the staged map (re-recording its manifest digest) and a dist chunk (re-sealing) one byte past the bound + and expect a refusal before any digest comparison — then rerun the three mutants. + *(low · tools/cli-package/symbolicate.mjs, tools/cli-package/symbolicate.test.mjs)* +- [ ] **No test shows that an injected anchor table is enforced.** *Nit.* *(round 3, TCA-9A2A-R3-2)* A mutant + that loops over the module's `ANCHORS` instead of the `anchors` argument survives: the only injected table + that reaches mapping, `[...ANCHORS, ANCHORS[0]]`, repeats a reached anchor. Production uses the default. + **Re-check:** add an anchor whose source the manifest maps and whose call site resolves in the source revision + but that no captured frame reaches, and expect `ANCHOR_UNMAPPED`. *(nit · tools/cli-package/symbolicate.test.mjs)* + ## Phase 2.6.5 deferrals — `CR-93` and `CR-95` (long half), recorded 2026-09-06; `CR-70`, `CR-94` (remainder) and the in-turn overflow continuation, recorded 2026-09-14; the errored-or-aborted turn residual, the pre-upgrade free-page bytes and the estimator's calibration, recorded 2026-09-18 **Phase closure — 2026-10-11:** W8 merged to `main` in PR #92 and Phase 2.6.5 is complete diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 36517a26..a2c28c47 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -3699,7 +3699,7 @@ No credential values or request captures belong in tracked evidence. | 9a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage #46/ADR-0111 unchanged, no new contract) — release maps, final build and symbolication:** fresh uncached upstream original-TypeScript maps, the sole prepack final CLI build, final trailer removal before hashes, map and private-graph staging outside public artifacts and caches, exact tarball-to-final-JS binding, and real packed CLI and engine frame symbolication with wrong JS/map/source and missing-map negative controls. Ordinary builds remain map-free. *Split again on 2026-10-19 into 9a.1 and 9a.2 below, which replace it as increments ([Plan shape](#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19)).* | | 9a.1 | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — candidate build and finalizer:** an owned isolated candidate copy, fresh uncached upstream maps, candidate-mode tsup selection with absent `sourcesContent`, the prepack finalizer (upstream and final map validation, repository-relative sources, exact trailer removal before hashes), owner-only private staging outside public artifacts and caches, closure/seal adaptation, the ordinary map-free control, the PR #95 archive/worker fold-ins, and a fail-closed `release.yml` publish guard that refuses any non-release candidate until 9d and Wave 4b custody exist. The receipt stays `compiled_package_test`. *Implemented on `development` on 2026-10-22 ([checkpoint](#wave-3-step-9a1-implementation-checkpoint--2026-10-22)); independently accepted at `cf97b7f1` on 2026-10-24 ([acceptance](#wave-3-step-9a1-acceptance--2026-10-24)).* | | 9a.2 | **Recorded split — manifest, symbolication and package-test receipt:** the private map manifest keyed by `dist/.js`, actual packed CLI and engine frame symbolication against the matching source revision, wrong JS/map/source and missing/malformed-upstream negative controls (product copies via `check:map-controls`, also a required uncached CI job), and the atomic `package_test` receipt flip across seal, binding, consumers, aggregate and both workflows. *Split on 2026-10-24 (maintainer-approved; ID coverage #46/ADR-0111 unchanged) into 9a.2a and 9a.2b below, which replace it as increments ([preflight decisions](#wave-3-step-9a2-preflight-decisions--2026-10-24)).* | -| 9a.2a | **Recorded split, 2026-10-24 (maintainer-approved) — decoder, private manifest and symbolication:** an in-house strict source-map decoder; strict mapping and range validation of upstream and final maps in the finalizer; the private map manifest (`staging/manifest.json`, keyed by `dist/.js`, with a per-source digest inventory and the commit's tree), sealed after the archive inspection; real packed CLI and `@relavium/core` frame capture through a tooling-only preload during an offline `relavium import`, run on the finalized `dist` that the inspection proves byte-equal to the tarball; symbolication of those frames against the matching source revision (clean: the checkout's Git objects; dirty, local only: the candidate copy), wired into the seal before the receipt is written. The receipt stays `compiled_package_test`. Security sitting. *Implemented on `development` on 2026-10-24; acceptance pending ([checkpoint](#wave-3-step-9a2a-implementation-checkpoint--2026-10-24)).* | +| 9a.2a | **Recorded split, 2026-10-24 (maintainer-approved) — decoder, private manifest and symbolication:** an in-house strict source-map decoder; strict mapping and range validation of upstream and final maps in the finalizer; the private map manifest (`staging/manifest.json`, keyed by `dist/.js`, with a per-source digest inventory and the commit's tree), sealed after the archive inspection; real packed CLI and `@relavium/core` frame capture through a tooling-only preload during an offline `relavium import`, run on the finalized `dist` that the inspection proves byte-equal to the tarball; symbolication of those frames against the matching source revision (clean: the checkout's Git objects; dirty, local only: the candidate copy), wired into the seal before the receipt is written. The receipt stays `compiled_package_test`. Security sitting. *Independently accepted on `development` at `a92a7602` on 2026-10-24 ([checkpoint](#wave-3-step-9a2a-implementation-checkpoint--2026-10-24), [acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)).* | | 9a.2b | **Recorded split, 2026-10-24 (maintainer-approved) — product map controls and the package-test receipt:** `check:map-controls` over product copies (wrong JS by line and column, wrong map, permuted sources, wrong source, missing, malformed and truncated upstream maps, a shifted valid upstream map, and the package-test producer's post-receipt rebuild refusals that ADR-0111 lists), run in `pnpm run ci` and as a required uncached CI job inside `ci`'s needs (the required contexts stay `ci` and `coverage`); then the atomic `package_test` flip across seal, binding, consumers, aggregate and both workflows, version 1 kept. Security sitting. | | 9b | **Recorded split — recipient custody:** synthetic-only isolated GnuPG controls (public-only home/keyring, pinned primary and encryption-subkey fingerprints, machine-readable refusal of missing, expired, revoked or unusable recipients), encrypt once and reuse the original ciphertext. Explicit crypto/custody security sitting. Fixture proof never claims production recipient or custody configuration. | | 9c | **Recorded split — offline release state controller:** permission-enforcing offline GitHub/npm controller, permissioned prebuild discovery and reservation, and every rerun, partial, conflict and expiry state, with no overwrite, rebuild or re-encryption on uncertain history. | @@ -9755,6 +9755,19 @@ hold in the capture process itself, not in a thread or subprocess the candidate contract, the preload's docblock and the SEC-9A2A-2 record say so, and that record notes its re-check (CRIT-9A2A-R2-1). -**Remaining gates.** Round 3 — one NEW complete cumulative round with no blocker, high or medium finding and no -regression, with the separate scoped security sitting at the same head and bar — then the full `pnpm run ci` and -the push CI. +Round 3 accepted the increment; see the acceptance below. + +### Wave 3 Step 9a.2a acceptance — 2026-10-24 + +**Step 9a.2a is independently accepted on `development` at `a92a7602` (2026-10-24)** under the Acceptance row as +amended on 2026-10-23. Round 3, a NEW independent complete cumulative review of `6277e375..a92a7602` with every +reviewer in its own worktree detached at that head, reported no blocker, high or medium finding and no +regression, and its separate security sitting, run at the same head as its own reviewer and report, was clean; +so were the plan, correctness and integration lenses and the critic. The plan reviewer ran the real producer at +the reviewed head: it exited 0 with a `compiled_package_test` receipt and `dirty: false`, the receipt's keys +were unchanged, neither the receipt nor the tarball held a map, the manifest, a frame, the per-source inventory or +a host path, and the sealed manifest (25 outputs, 404 sources) and the symbolication of all five anchors (about a +second) matched this checkpoint. The round's three verified findings (one low, two nits) are recorded in +[deferred-tasks.md](../deferred-tasks.md#wave-3-step-9a2a--recorded-low-findings-2026-10-24); the wording nit is +corrected in the acceptance commit, which changes no code, and the two test gaps stay open there. The receipt +stays `compiled_package_test`; Step 9a.2b is next. From 436d7c67c6a0acdb79995fedc6bda8b48c4d5f16 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sat, 24 Oct 2026 18:05:14 +0300 Subject: [PATCH 43/97] docs(repo): record the Step 9a.2b preflight decisions A read-only preflight of Step 9a.2b (five readers and a synthesis at f65999a6) mapped the product map controls, the post-receipt rebuild refusals, the package_test flip and the CI wiring. The maintainer approved every recommendation on 2026-10-24: - one real producePackage run drives the controls through its ports, with every private copy under the staging; - sealPackage refuses a re-seal with PACKED_CLI_ALREADY_SEALED before the inspection and any capture; - a new verifySealedDist checks the whole dist inventory, every output digest and the tarball digest before the receipt is written; - release.yml gets no map-controls job (9d owns that graph); - one increment with ordered commits and one acceptance gate. Refs: ADR-0111, #46 Co-Authored-By: Claude Opus 5.5 --- .../phase-2.5.5-hardening-and-remediation.md | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index a2c28c47..a7dcc8e6 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -3700,7 +3700,7 @@ No credential values or request captures belong in tracked evidence. | 9a.1 | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — candidate build and finalizer:** an owned isolated candidate copy, fresh uncached upstream maps, candidate-mode tsup selection with absent `sourcesContent`, the prepack finalizer (upstream and final map validation, repository-relative sources, exact trailer removal before hashes), owner-only private staging outside public artifacts and caches, closure/seal adaptation, the ordinary map-free control, the PR #95 archive/worker fold-ins, and a fail-closed `release.yml` publish guard that refuses any non-release candidate until 9d and Wave 4b custody exist. The receipt stays `compiled_package_test`. *Implemented on `development` on 2026-10-22 ([checkpoint](#wave-3-step-9a1-implementation-checkpoint--2026-10-22)); independently accepted at `cf97b7f1` on 2026-10-24 ([acceptance](#wave-3-step-9a1-acceptance--2026-10-24)).* | | 9a.2 | **Recorded split — manifest, symbolication and package-test receipt:** the private map manifest keyed by `dist/.js`, actual packed CLI and engine frame symbolication against the matching source revision, wrong JS/map/source and missing/malformed-upstream negative controls (product copies via `check:map-controls`, also a required uncached CI job), and the atomic `package_test` receipt flip across seal, binding, consumers, aggregate and both workflows. *Split on 2026-10-24 (maintainer-approved; ID coverage #46/ADR-0111 unchanged) into 9a.2a and 9a.2b below, which replace it as increments ([preflight decisions](#wave-3-step-9a2-preflight-decisions--2026-10-24)).* | | 9a.2a | **Recorded split, 2026-10-24 (maintainer-approved) — decoder, private manifest and symbolication:** an in-house strict source-map decoder; strict mapping and range validation of upstream and final maps in the finalizer; the private map manifest (`staging/manifest.json`, keyed by `dist/.js`, with a per-source digest inventory and the commit's tree), sealed after the archive inspection; real packed CLI and `@relavium/core` frame capture through a tooling-only preload during an offline `relavium import`, run on the finalized `dist` that the inspection proves byte-equal to the tarball; symbolication of those frames against the matching source revision (clean: the checkout's Git objects; dirty, local only: the candidate copy), wired into the seal before the receipt is written. The receipt stays `compiled_package_test`. Security sitting. *Independently accepted on `development` at `a92a7602` on 2026-10-24 ([checkpoint](#wave-3-step-9a2a-implementation-checkpoint--2026-10-24), [acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)).* | -| 9a.2b | **Recorded split, 2026-10-24 (maintainer-approved) — product map controls and the package-test receipt:** `check:map-controls` over product copies (wrong JS by line and column, wrong map, permuted sources, wrong source, missing, malformed and truncated upstream maps, a shifted valid upstream map, and the package-test producer's post-receipt rebuild refusals that ADR-0111 lists), run in `pnpm run ci` and as a required uncached CI job inside `ci`'s needs (the required contexts stay `ci` and `coverage`); then the atomic `package_test` flip across seal, binding, consumers, aggregate and both workflows, version 1 kept. Security sitting. | +| 9a.2b | **Recorded split, 2026-10-24 (maintainer-approved) — product map controls and the package-test receipt:** `check:map-controls` over product copies (wrong JS by line and column, wrong map, permuted sources, wrong source, missing, malformed and truncated upstream maps, a shifted valid upstream map, and the package-test producer's post-receipt rebuild refusals that ADR-0111 lists), run in `pnpm run ci` and as a required uncached CI job inside `ci`'s needs (the required contexts stay `ci` and `coverage`); then the atomic `package_test` flip across seal, binding, consumers, aggregate and both workflows, version 1 kept. Security sitting. *Preflight decisions approved on 2026-10-24 ([decisions](#wave-3-step-9a2b-preflight-decisions--2026-10-24)).* | | 9b | **Recorded split — recipient custody:** synthetic-only isolated GnuPG controls (public-only home/keyring, pinned primary and encryption-subkey fingerprints, machine-readable refusal of missing, expired, revoked or unusable recipients), encrypt once and reuse the original ciphertext. Explicit crypto/custody security sitting. Fixture proof never claims production recipient or custody configuration. | | 9c | **Recorded split — offline release state controller:** permission-enforcing offline GitHub/npm controller, permissioned prebuild discovery and reservation, and every rerun, partial, conflict and expiry state, with no overwrite, rebuild or re-encryption on uncertain history. | | 9d | **Recorded split — workflow, runbook and whole-Step-9 landing:** least-privilege actual workflow dependency graph, required artifact/platform aggregation and exact-artifact consumers, lost-staging refusal and the release runbook update. The whole-Step-9 cumulative review retains all 9a–9d changes with preservation traces into Step 8, and the dedicated archive/crypto security sitting closes the step. | @@ -9771,3 +9771,28 @@ second) matched this checkpoint. The round's three verified findings (one low, t [deferred-tasks.md](../deferred-tasks.md#wave-3-step-9a2a--recorded-low-findings-2026-10-24); the wording nit is corrected in the acceptance commit, which changes no code, and the two test gaps stay open there. The receipt stays `compiled_package_test`; Step 9a.2b is next. + +### Wave 3 Step 9a.2b preflight decisions — 2026-10-24 + +A read-only preflight (five readers and a synthesis at `f65999a6`; the detailed record and a Turkish decision +register are local and Git-ignored) mapped every place Step 9a.2b touches. The maintainer approved each +recommendation on 2026-10-24; no new ADR or runtime dependency follows. The earlier 9a.2 rulings stand: the +three post-receipt rebuild refusals, the receipt keeping version 1 and its twelve fields with only `mode` changing +to `package_test` and `compiled_package_test` refused, the standalone runner `tools/cli-package/map-controls.mjs`, +and a `map-controls` CI job shaped like `pack-cli` (uncached, no upload, at most 20 minutes, inside `ci`'s needs +with a success guard; the required contexts stay `ci` and `coverage`). + +| Decision | Ruling | +| --- | --- | +| Sealed product | One real `producePackage` run, driven through its existing ports: the controls run inside the seal callback while the private staging still exists, mutate the owned candidate and staging copies in place and restore each file by digest; every private copy stays under the staging, which is removed first on success, failure and interrupt. `produce.mjs`'s cleanup and interrupt handling stay the one owner. | +| Re-seal | `sealPackage` refuses with `PACKED_CLI_ALREADY_SEALED` at its entry, before the inspection and any capture, when the artifact directory holds anything besides the one tarball or the staging already holds a manifest; a receipt-write race maps to the same code. | +| Rebuilt `dist` | A new `verifySealedDist` compares the whole `dist` inventory, every output's digest and the tarball's digest with the in-memory manifest (`PACKED_CLI_MANIFEST_DIST_DRIFT`, `PACKED_CLI_MANIFEST_TARBALL_CHANGED`); the seal calls it immediately before the receipt write, and the control calls it after a real rebuild. | +| Release workflow | `release.yml` gets no map-controls job: the interim guard refuses every release-workflow artifact until 9d and Wave 4b, and 9d owns the workflow graph. A manual dry-run's `package_test` artifact is backed by the required `ci` run at the same commit, and the contract says so. | +| Increments | One increment with ordered commits — the seal refusals, the runner, the CI job and its pins, then the flip, then the documents — under one acceptance gate and one security sitting; the flip never exists in a tree without the controls before it. | + +Engineering defaults recorded with the rulings: each control asserts its exact refusal code and its stage markers +(so a refusal from the wrong layer cannot pass); wrong source runs both clean (`SOURCE_REVISION`) and dirty +(`ANCHOR_UNMAPPED`); truncation covers a byte cut and a cut inside a VLQ value; the JS and map controls run on every +anchor-bearing chunk; permuted sources rotate with no fixed point; the shifted upstream map is checked to be +strictly valid before use; the runner prints one JSON summary with codes and outcomes only — no path, frame, map +byte or child output — and compares bytes by digest; it runs in `pnpm run ci` right after `smoke:cli-package`. From a45f814a565c6ec85a0f9ad005be0096261f39dc Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sat, 24 Oct 2026 18:10:02 +0300 Subject: [PATCH 44/97] feat(repo): refuse a re-seal and a changed dist before the receipt Step 9a.2b, first commit (ADR-0111's post-receipt rebuild refusal for the package-test producer, as the 2026-10-24 preflight ruled): - sealPackage refuses with PACKED_CLI_ALREADY_SEALED at its entry, before the inspection and any capture, when the artifact directory holds anything besides the one tarball or the staging already holds a manifest (checked without following a link). A receipt written by another seal meanwhile maps to the same code instead of a raw EEXIST. - A new verifySealedDist checks the whole dist inventory, every output digest and the tarball digest against the manifest just sealed (PACKED_CLI_MANIFEST_DIST_DRIFT, PACKED_CLI_MANIFEST_TARBALL_CHANGED). The seal calls it after symbolication, immediately before the receipt, so a dist changed after the inspection refuses, also in a chunk no frame lands in. - Tests: re-seal after a receipt, a staged manifest and a dangling link to one, the receipt race, the verify port's order and arguments, each drift case of verifySealedDist, and a second prepack into a sealed staging (STAGING_NOT_EMPTY before any build). The receipt mode is unchanged; the flip comes after the controls. Refs: ADR-0111, #46 Co-Authored-By: Claude Opus 5.5 --- tools/cli-package/manifest.mjs | 37 ++++++++++++ tools/cli-package/manifest.test.mjs | 77 +++++++++++++++++++++++- tools/cli-package/prepack.test.mjs | 25 ++++++++ tools/cli-package/seal.mjs | 62 +++++++++++++++---- tools/cli-package/seal.test.mjs | 93 ++++++++++++++++++++++++++--- 5 files changed, 274 insertions(+), 20 deletions(-) diff --git a/tools/cli-package/manifest.mjs b/tools/cli-package/manifest.mjs index 0c56ec78..eb020cca 100644 --- a/tools/cli-package/manifest.mjs +++ b/tools/cli-package/manifest.mjs @@ -7,6 +7,7 @@ * staging; Step 9b carries it into ciphertext. */ import { Buffer } from 'node:buffer'; +import { readdirSync } from 'node:fs'; import { createRequire } from 'node:module'; import { basename, join } from 'node:path'; import { isDeepStrictEqual } from 'node:util'; @@ -283,6 +284,42 @@ export async function sealPrivateManifest({ return manifest; } +/** + * Verify that the finalized `dist` and the tarball are still the ones `manifest` records: exactly its + * outputs, each with its digest, and the tarball's digest (ADR-0111's post-receipt rebuild refusal; + * Step 9a.2b). The seal runs it with the manifest it has just sealed, after symbolication and + * immediately before the receipt is written, so a `dist` rebuilt or changed after the inspection — in + * a chunk no frame lands in as well — refuses. + * @param {{candidate: string, manifest: PrivateManifest, tarball: string}} options + */ +export function verifySealedDist({ candidate, manifest, tarball }) { + const cli = join(candidate, 'apps', 'cli'); + let names; + try { + names = readdirSync(join(cli, 'dist')).sort(); + } catch { + refuse('DIST_DRIFT'); + } + const expected = manifest.outputs.map((output) => basename(output.path)).sort(); + if (!isDeepStrictEqual(names, expected)) refuse('DIST_DRIFT'); + for (const output of manifest.outputs) { + let bytes; + try { + bytes = readOrdinaryFile(join(cli, ...output.path.split('/')), 8 * 1024 * 1024); + } catch { + refuse('DIST_DRIFT'); + } + if (sha256(bytes) !== output.sha256) refuse('DIST_DRIFT'); + } + let packed; + try { + packed = readOrdinaryFile(tarball, 16 * 1024 * 1024); + } catch { + refuse('TARBALL_CHANGED'); + } + if (sha256(packed) !== manifest.tarballSha256) refuse('TARBALL_CHANGED'); +} + /** * Read and validate the sealed manifest from `staging`. * @param {string} staging diff --git a/tools/cli-package/manifest.test.mjs b/tools/cli-package/manifest.test.mjs index e08b8627..2dbe15a2 100644 --- a/tools/cli-package/manifest.test.mjs +++ b/tools/cli-package/manifest.test.mjs @@ -1,7 +1,15 @@ import assert from 'node:assert/strict'; import { Buffer } from 'node:buffer'; import { createRequire } from 'node:module'; -import { existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'; +import { + existsSync, + mkdirSync, + readFileSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; import { join, posix } from 'node:path'; import { test } from 'node:test'; import { fileURLToPath } from 'node:url'; @@ -27,6 +35,7 @@ import { ManifestError, readPrivateManifest, sealPrivateManifest, + verifySealedDist, } from './manifest.mjs'; const refuses = (code) => (error) => error instanceof ManifestError && error.code === code; @@ -392,6 +401,72 @@ test('every manifest read is bounded', async () => { assert.throws(() => readPrivateManifest(empty), refuses('UNREADABLE')); }); +/** A sealed fixture whose manifest records the digest of an actual tarball file. */ +async function sealedWithTarball() { + const fixture = await candidate(); + const artifact = fresh('artifact'); + mkdirSync(artifact); + const tarball = join(artifact, 'relavium-0.1.1.tgz'); + writeFileSync(tarball, 'synthetic tarball bytes'); + fixture.archive.sha256 = digest(readFileSync(tarball)); + return { fixture, tarball, manifest: await seal(fixture) }; +} + +test('the dist and the tarball are verified against the manifest just sealed', async () => { + const sealed = await sealedWithTarball(); + verifySealedDist({ + candidate: sealed.fixture.root, + manifest: sealed.manifest, + tarball: sealed.tarball, + }); + for (const [label, change, code] of [ + [ + 'one more byte in a chunk no anchor lies in', + (fixture) => { + const path = join(fixture.cli, 'dist', 'index.js'); + writeFileSync(path, `${readFileSync(path, 'utf8')} `); + }, + 'DIST_DRIFT', + ], + [ + 'an extra file', + (fixture) => writeFileSync(join(fixture.cli, 'dist', 'extra.js'), ''), + 'DIST_DRIFT', + ], + [ + 'a removed chunk', + (fixture) => rmSync(join(fixture.cli, 'dist', coreChunk(fixture))), + 'DIST_DRIFT', + ], + [ + 'a chunk replaced by a link to the same bytes', + (fixture) => { + const path = join(fixture.cli, 'dist', 'index.js'); + const copy = join(fixture.work, 'index.js'); + writeFileSync(copy, readFileSync(path)); + rmSync(path); + symlinkSync(copy, path); + }, + 'DIST_DRIFT', + ], + ['no dist', (fixture) => rmSync(join(fixture.cli, 'dist'), { recursive: true }), 'DIST_DRIFT'], + [ + 'a replaced tarball', + (_fixture, tarball) => writeFileSync(tarball, 'other bytes'), + 'TARBALL_CHANGED', + ], + ['a missing tarball', (_fixture, tarball) => rmSync(tarball), 'TARBALL_CHANGED'], + ]) { + const { fixture, tarball, manifest } = await sealedWithTarball(); + change(fixture, tarball); + assert.throws( + () => verifySealedDist({ candidate: fixture.root, manifest, tarball }), + refuses(code), + label, + ); + } +}); + test('a refusal is a ManifestError naming only its fixed code', () => { const empty = fresh('no-manifest'); mkdirSync(empty); diff --git a/tools/cli-package/prepack.test.mjs b/tools/cli-package/prepack.test.mjs index ee86acba..4a8046d1 100644 --- a/tools/cli-package/prepack.test.mjs +++ b/tools/cli-package/prepack.test.mjs @@ -7,6 +7,7 @@ import { mkdirSync, mkdtempSync, readFileSync, + readdirSync, realpathSync, rmSync, symlinkSync, @@ -119,6 +120,30 @@ test('an unusable staging directory refuses before anything is built', () => { } }); +test('a second prepack into a sealed staging refuses before anything is built', () => { + // The staging of a sealed candidate: the finalizer's record, the metafile, the staged maps and the + // private manifest (ADR-0111's post-receipt rebuild refusal; Step 9a.2b). + const candidate = fixture(); + const staging = privateStaging(); + const sealed = { + 'finalized.json': '{"version":1}', + 'metafile-esm.json': '{"outputs":{}}', + 'manifest.json': '{"version":1}', + }; + for (const [name, text] of Object.entries(sealed)) writeFileSync(join(staging, name), text); + mkdirSync(join(staging, 'maps')); + writeFileSync(join(staging, 'maps', 'index.js.map'), '{"version":3}'); + const result = prepack(candidate, { staging }); + assert.equal(result.status, 1); + assert.match(result.stderr, /CLI candidate finalization refused: STAGING_NOT_EMPTY/u); + assert.equal(existsSync(candidate.marker), false); + assert.deepEqual(readdirSync(staging).sort(), [...Object.keys(sealed), 'maps'].sort()); + for (const [name, text] of Object.entries(sealed)) { + assert.equal(readFileSync(join(staging, name), 'utf8'), text); + } + assert.equal(readFileSync(join(staging, 'maps', 'index.js.map'), 'utf8'), '{"version":3}'); +}); + test('a failed build stops before the closure gate or finalization', () => { const candidate = fixture({ buildExit: 3 }); const staging = privateStaging(); diff --git a/tools/cli-package/seal.mjs b/tools/cli-package/seal.mjs index 8b7fe98f..1d37a0c2 100644 --- a/tools/cli-package/seal.mjs +++ b/tools/cli-package/seal.mjs @@ -6,12 +6,12 @@ */ import assert from 'node:assert/strict'; import { createRequire } from 'node:module'; -import { readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { lstatSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; import { basename, isAbsolute, join } from 'node:path'; import { inspectPackageArchive } from './archive.mjs'; import { originFromEnvironment, validatePackageReceipt } from './binding.mjs'; import { flatFiles, oneTarball, readOrdinaryFile, sha256 } from './files.mjs'; -import { sealPrivateManifest } from './manifest.mjs'; +import { MANIFEST_FILE, sealPrivateManifest, verifySealedDist } from './manifest.mjs'; import { symbolicateCandidate } from './symbolicate.mjs'; export function producerNodeMajor(root) { @@ -93,20 +93,38 @@ export function finalizedPublicFiles(root, staging) { return files; } +/** Whether anything exists at `path`, without following a link. */ +function present(path) { + try { + lstatSync(path); + return true; + } catch (error) { + if (error?.code === 'ENOENT') return false; + throw error; + } +} + /** * Inspect the tarball, then seal the candidate's private map manifest, then symbolicate actual packed - * frames against it, and only then write the receipt: a refusal at any step leaves no receipt. + * frames against it, then verify that `dist` and the tarball are still the manifest's, and only then + * write the receipt: a refusal at any step leaves no receipt. A candidate is sealed once: a receipt + * beside the tarball or a manifest in the staging refuses before anything is read or run. * @param {string} artifactDirectory the public artifact directory holding exactly one tarball * @param {{repository: string, staging: string, source: string, dirty: boolean, checkout: string, * work: string, ports?: {manifest?: typeof sealPrivateManifest, - * symbolicate?: typeof symbolicateCandidate}}} candidate the candidate root and its private - * staging, the original checkout's identity and object store, and a private work directory + * symbolicate?: typeof symbolicateCandidate, verify?: typeof verifySealedDist}}} candidate the + * candidate root and its private staging, the original checkout's identity and object store, and a + * private work directory */ export async function sealPackage( artifactDirectory, { repository, staging, source, dirty, checkout, work, ports = {} }, ) { - const { manifest = sealPrivateManifest, symbolicate = symbolicateCandidate } = ports; + const { + manifest = sealPrivateManifest, + symbolicate = symbolicateCandidate, + verify = verifySealedDist, + } = ports; const tarball = oneTarball(artifactDirectory); assert.match(source, /^[0-9a-f]{40}$/u, 'PACKED_CLI_SOURCE_IDENTITY'); assert.equal(typeof dirty, 'boolean', 'PACKED_CLI_SOURCE_IDENTITY'); @@ -117,6 +135,11 @@ export async function sealPackage( isAbsolute(work), 'PACKED_CLI_SEAL_INPUTS', ); + // ADR-0111's post-receipt rebuild refusal (Step 9a.2b): a sealed candidate is never sealed again. + assert.ok( + readdirSync(artifactDirectory).length === 1 && !present(join(staging, MANIFEST_FILE)), + 'PACKED_CLI_ALREADY_SEALED', + ); const archive = inspectPackageArchive(readOrdinaryFile(tarball, 16 * 1024 * 1024), { expectedFiles: finalizedPublicFiles(repository, staging), expectedPackage: expectedPackManifest(repository), @@ -126,8 +149,17 @@ export async function sealPackage( assert.equal(pnpm, 'pnpm@9.12.3'); // Only now, with the packed bytes proven to be the finalized ones, the private manifest is sealed; // actual packed frames are then symbolicated against it before any receipt exists. - await manifest({ candidate: repository, staging, checkout, source, dirty, archive }); + const sealed = await manifest({ + candidate: repository, + staging, + checkout, + source, + dirty, + archive, + }); await symbolicate({ candidate: repository, staging, checkout, work, archive }); + // The capture ran the candidate: its dist and the tarball must still be the ones just sealed. + await verify({ candidate: repository, manifest: sealed, tarball }); const receipt = { version: 1, mode: 'compiled_package_test', @@ -148,10 +180,16 @@ export async function sealPackage( nodeMajor: producerNodeMajor(repository), allowLocal: origin.kind === 'local', }); - writeFileSync( - join(artifactDirectory, 'package-test-receipt.json'), - JSON.stringify(receipt, null, 2) + '\n', - { flag: 'wx' }, - ); + try { + writeFileSync( + join(artifactDirectory, 'package-test-receipt.json'), + JSON.stringify(receipt, null, 2) + '\n', + { flag: 'wx' }, + ); + } catch (error) { + // Another seal wrote its receipt meanwhile. + if (error?.code === 'EEXIST') assert.fail('PACKED_CLI_ALREADY_SEALED'); + throw error; + } return receipt; } diff --git a/tools/cli-package/seal.test.mjs b/tools/cli-package/seal.test.mjs index bd527d44..9483ff6b 100644 --- a/tools/cli-package/seal.test.mjs +++ b/tools/cli-package/seal.test.mjs @@ -371,14 +371,21 @@ const receiptPath = (artifact) => join(artifact, 'package-test-receipt.json'); * The seal with recording stand-ins for the private manifest and symbolication, whose own tests * (manifest.test.mjs, symbolicate.test.mjs) run them for real. */ +/** What the manifest stand-in returns as the sealed manifest. */ +const SEALED = Object.freeze({ sealed: 'manifest' }); + function sealWith(artifact, candidate, calls = []) { return sealPackage(artifact, { checkout: candidate.repository, work: join(dirname(candidate.staging), 'work'), ...candidate, ports: { - manifest: async (options) => void calls.push(['manifest', options]), + manifest: async (options) => { + calls.push(['manifest', options]); + return SEALED; + }, symbolicate: async (options) => void calls.push(['symbolicate', options]), + verify: async (options) => void calls.push(['verify', options]), }, }); } @@ -489,9 +496,16 @@ test('the seal seals the private manifest after the inspection and symbolicates ); assert.deepEqual( calls.map(([step]) => step), - ['manifest', 'symbolicate'], + ['manifest', 'symbolicate', 'verify'], ); - const [[, manifest], [, symbolicate]] = calls; + const [[, manifest], [, symbolicate], [, verify]] = calls; + // The dist and tarball check receives the manifest just sealed, in memory, and the inspected tarball. + assert.deepEqual(verify, { + candidate: root, + manifest: SEALED, + tarball: join(artifact, 'relavium-0.1.1.tgz'), + }); + assert.equal(verify.manifest, SEALED); // Both steps receive the inspected archive: the bytes the receipt then records. assert.equal(manifest.archive.sha256, receipt.tarballSha256); assert.deepEqual(manifest.archive.files, receipt.files); @@ -514,8 +528,9 @@ test('the seal seals the private manifest after the inspection and symbolicates }); }); -test('a refused manifest or symbolication leaves no receipt, and an unproven archive reaches neither', async () => { - for (const failing of ['manifest', 'symbolicate']) { +test('a refused manifest, symbolication or dist check leaves no receipt, and an unproven archive reaches none', async () => { + const order = ['manifest', 'symbolicate', 'verify']; + for (const failing of order) { await withCandidate(async ({ root, staging, artifact }) => { const steps = []; const refusal = new Error(`synthetic ${failing} refusal`); @@ -532,12 +547,16 @@ test('a refused manifest or symbolication leaves no receipt, and an unproven arc dirty: false, checkout: root, work: join(dirname(staging), 'work'), - ports: { manifest: step('manifest'), symbolicate: step('symbolicate') }, + ports: { + manifest: step('manifest'), + symbolicate: step('symbolicate'), + verify: step('verify'), + }, }), (error) => error === refusal, ), ); - assert.deepEqual(steps, failing === 'manifest' ? ['manifest'] : ['manifest', 'symbolicate']); + assert.deepEqual(steps, order.slice(0, order.indexOf(failing) + 1)); assert.equal(existsSync(receiptPath(artifact)), false); }); } @@ -555,6 +574,65 @@ test('a refused manifest or symbolication leaves no receipt, and an unproven arc }); }); +test('a sealed candidate is never sealed again, and nothing is read or run first', async () => { + // A receipt beside the tarball: the candidate was sealed. + await withCandidate(async ({ root, staging, artifact }) => { + await withOrigin({}, () => + sealWith(artifact, { repository: root, staging, source: SOURCE, dirty: false }), + ); + const receipt = readFileSync(receiptPath(artifact)); + const calls = []; + await withOrigin({}, () => + assert.rejects( + sealWith(artifact, { repository: root, staging, source: SOURCE, dirty: false }, calls), + { message: 'PACKED_CLI_ALREADY_SEALED' }, + ), + ); + assert.deepEqual(calls, []); + assert.deepEqual(readFileSync(receiptPath(artifact)), receipt); + }); + // A manifest in the staging, even a dangling link, refuses the same way. + for (const plant of [ + (path) => writeFileSync(path, '{}'), + (path) => symlinkSync(join(dirname(path), 'absent'), path), + ]) { + await withCandidate(async ({ root, staging, artifact }) => { + plant(join(staging, 'manifest.json')); + const calls = []; + await withOrigin({}, () => + assert.rejects( + sealWith(artifact, { repository: root, staging, source: SOURCE, dirty: false }, calls), + { message: 'PACKED_CLI_ALREADY_SEALED' }, + ), + ); + assert.deepEqual(calls, []); + assert.equal(existsSync(receiptPath(artifact)), false); + }); + } + // Another seal that writes its receipt meanwhile wins, and this one refuses with the same code. + await withCandidate(async ({ root, staging, artifact }) => { + await withOrigin({}, () => + assert.rejects( + sealPackage(artifact, { + repository: root, + staging, + source: SOURCE, + dirty: false, + checkout: root, + work: join(dirname(staging), 'work'), + ports: { + manifest: async () => SEALED, + symbolicate: async () => writeFileSync(receiptPath(artifact), 'the other seal'), + verify: async () => {}, + }, + }), + { message: 'PACKED_CLI_ALREADY_SEALED' }, + ), + ); + assert.equal(readFileSync(receiptPath(artifact), 'utf8'), 'the other seal'); + }); +}); + test('both steps receive the candidate, the original checkout and the dirty flag as given', async () => { await withCandidate(async ({ root, staging, artifact }) => { const calls = []; @@ -570,6 +648,7 @@ test('both steps receive the candidate, the original checkout and the dirty flag ports: { manifest: async (options) => void calls.push(options), symbolicate: async (options) => void calls.push(options), + verify: async () => {}, }, }), ); From 6d98133683f481ba9998edbae1322205c5f34802 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sat, 24 Oct 2026 18:21:17 +0300 Subject: [PATCH 45/97] feat(repo): run the product map controls on one real production Step 9a.2b, second commit: tools/cli-package/map-controls.mjs, the standalone runner the 2026-10-24 preflight approved, and its script check:map-controls. One real package-test production runs at the checked-out commit through producePackage's emit and seal ports. Inside the seal, after the baseline receipt and while the private staging exists, every control corrupts one product copy (the owned candidate or a copy of its staging), requires its exact refusal and restores the file by digest: - a second prepack into the sealed staging (STAGING_NOT_EMPTY, before any build, nothing changed) and a re-seal of the artifact directory or of the sealed staging (ALREADY_SEALED, before any capture); - on every anchor-bearing chunk: wrong JS by line and by column, a map from a build one line apart and rotated map sources (SYMBOLICATION_ANCHOR_UNMAPPED, after re-recording every digest so only the mapping layer can refuse); a re-recorded unchanged copy is accepted; - a wrong source revision, clean (SOURCE_REVISION) and dirty (ANCHOR_UNMAPPED); - the sealed dist verifies; a changed chunk no frame lands in and a real rebuild from a changed source refuse (MANIFEST_DIST_DRIFT); - upstream maps: missing, malformed shape, an out-of-range index and a value cut mid-VLQ refuse in the finalizer after the bundler silently succeeded (one build, nothing finalized, staged or packed); a byte-truncated map fails the bundler's own build; a shifted but strictly valid map is accepted there and refused by symbolication; - a restoration production seals again. Every copy stays under the staging, which the producer removes first; the developer checkout must be unchanged before and after. One JSON summary of ids, expected and observed codes is printed, never a path, frame or map byte. On this host the 24 controls pass in about 30 s. map-controls.test.mjs covers the pure helpers (the VLQ encoder against the strict decoder, the valid column shift, rotation without a fixed point, the line and column shifts, a cut inside a value). Refs: ADR-0111, #46 Co-Authored-By: Claude Opus 5.5 --- package.json | 3 +- tools/cli-package/map-controls.mjs | 692 ++++++++++++++++++++++++ tools/cli-package/map-controls.test.mjs | 130 +++++ 3 files changed, 824 insertions(+), 1 deletion(-) create mode 100644 tools/cli-package/map-controls.mjs create mode 100644 tools/cli-package/map-controls.test.mjs diff --git a/package.json b/package.json index 83fded56..8cb50213 100644 --- a/package.json +++ b/package.json @@ -36,12 +36,13 @@ "smoke:budget-replay": "node tools/budget-replay-compat/check.mjs", "check:wire-mutations": "node tools/wire-conformance/check.mjs", "check:usage-mutations": "node tools/usage-coverage-controls/check.mjs", - "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/check.test.mjs tools/cli-package/paths.test.mjs tools/cli-package/source-map.test.mjs tools/cli-package/manifest.test.mjs tools/cli-package/symbolicate.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", + "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/check.test.mjs tools/cli-package/paths.test.mjs tools/cli-package/source-map.test.mjs tools/cli-package/manifest.test.mjs tools/cli-package/symbolicate.test.mjs tools/cli-package/map-controls.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", "check:fs-mutations": "node tools/fs-policy-controls/check.mjs", "check:secrets-mutations": "node tools/secrets-controls/check.mjs", "smoke:cli-bootstrap": "node tools/cli-bootstrap/check.mjs", "package:cli:pack": "node tools/cli-package/pack.mjs", "package:cli:consume": "node tools/cli-package/consume.mjs", + "check:map-controls": "node tools/cli-package/map-controls.mjs", "smoke:cli-package": "node tools/cli-package/check.mjs" }, "devDependencies": { diff --git a/tools/cli-package/map-controls.mjs b/tools/cli-package/map-controls.mjs new file mode 100644 index 00000000..b6eecfc0 --- /dev/null +++ b/tools/cli-package/map-controls.mjs @@ -0,0 +1,692 @@ +/** + * Product map controls (ADR-0111; packaging.md "Product map controls — Step 9a.2b"; the 2026-10-24 + * preflight rulings). One real package-test production runs at the checked-out commit. Inside its + * seal, after the baseline receipt and while the private staging still exists, every named negative + * control corrupts one product copy — the producer's own owned candidate tree, or a copy of its private + * staging — and requires that control's exact refusal; each corrupted file is restored and checked by + * digest, and a restoration production must then seal again. Every copy a control makes stays under + * the staging, which the producer removes first on success, failure and interrupt; the developer's + * checkout is never touched. One JSON summary of control ids, expected and observed codes and outcomes + * is printed — never a path, frame, position, map byte or child output. + */ +import assert from 'node:assert/strict'; +import { Buffer } from 'node:buffer'; +import { + chmodSync, + cpSync, + lstatSync, + mkdirSync, + mkdtempSync, + readdirSync, + realpathSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { sourceIdentity } from './candidate.mjs'; +import { CANDIDATE_STAGING_ENV } from './finalize.mjs'; +import { readOrdinaryFile, sha256 } from './files.mjs'; +import { readPrivateManifest, sealPrivateManifest, verifySealedDist } from './manifest.mjs'; +import { emitFreshPackage } from './pipeline.mjs'; +import { producePackage } from './produce.mjs'; +import { productionFailure } from './production-failure.mjs'; +import { sealPackage } from './seal.mjs'; +import { + assertGeneratedRange, + assertOriginalRange, + decodeMappings, + lineLengths, +} from './source-map.mjs'; +import { ANCHORS, symbolicateCandidate } from './symbolicate.mjs'; + +const BASE64 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; +const MIB = 1024 * 1024; + +/** Base64 VLQ of one signed value, as a source map writer encodes it. */ +function vlq(value) { + let rest = value < 0 ? -value * 2 + 1 : value * 2; + let out = ''; + do { + let digit = rest % 32; + rest = Math.floor(rest / 32); + if (rest > 0) digit += 32; + out += BASE64[digit]; + } while (rest > 0); + return out; +} + +/** + * Encode decoded segments (as `decodeMappings` returns them) back into `mappings`. + * @param {import('./source-map.mjs').Segment[][]} lines + */ +export function encodeMappings(lines) { + let source = 0; + let originalLine = 0; + let originalColumn = 0; + let name = 0; + return lines + .map((segments) => { + let column = 0; + return segments + .map((segment) => { + let out = vlq(segment.generatedColumn - column); + column = segment.generatedColumn; + if (segment.source !== undefined) { + out += + vlq(segment.source - source) + + vlq(segment.originalLine - originalLine) + + vlq(segment.originalColumn - originalColumn); + source = segment.source; + originalLine = segment.originalLine; + originalColumn = segment.originalColumn; + if (segment.name !== undefined) { + out += vlq(segment.name - name); + name = segment.name; + } + } + return out; + }) + .join(','); + }) + .join(';'); +} + +/** + * A map shifted while staying valid: every original column after a line's start moves one to the left, + * so every position stays inside its text and only symbolication can tell. + * @param {string} mappings @param {{sources: number, names: number}} counts + */ +export function shiftOriginalColumns(mappings, counts) { + const lines = decodeMappings(mappings, counts); + let moved = 0; + for (const segments of lines) { + for (const segment of segments) { + if (segment.originalColumn !== undefined && segment.originalColumn > 0) { + segment.originalColumn -= 1; + moved += 1; + } + } + } + assert.ok(moved > 0, 'PACKED_CLI_MAP_CONTROL_NOTHING_SHIFTED'); + return encodeMappings(lines); +} + +/** Sources rotated by one: with two or more distinct sources, no source keeps its index. */ +export function rotateSources(sources) { + assert.ok( + sources.length >= 2 && new Set(sources).size === sources.length, + 'PACKED_CLI_MAP_CONTROL_SOURCES', + ); + return [...sources.slice(1), sources[0]]; +} + +/** The JS one line down: a line break after the hashbang line, or at the start. */ +export function shiftLines(text) { + if (!text.startsWith('#!')) return `\n${text}`; + const end = text.indexOf('\n'); + assert.ok(end > 0, 'PACKED_CLI_MAP_CONTROL_HASHBANG'); + return `${text.slice(0, end + 1)}\n${text.slice(end + 1)}`; +} + +/** The JS one column right: a space before every line but a hashbang line. */ +export function shiftColumns(text) { + return text + .split('\n') + .map((line, index) => (index === 0 && line.startsWith('#!') ? line : ` ${line}`)) + .join('\n'); +} + +/** `mappings` cut right after the first continuation digit, so the value it starts never ends. */ +export function cutInsideValue(mappings) { + for (let index = 0; index < mappings.length; index += 1) { + if (BASE64.indexOf(mappings[index]) >= 32) return mappings.slice(0, index + 1); + } + assert.fail('PACKED_CLI_MAP_CONTROL_NO_CONTINUATION'); +} + +/** An owner-only directory, created fresh. */ +function privateDirectory(path) { + mkdirSync(path, { recursive: true, mode: 0o700 }); + chmodSync(path, 0o700); + return path; +} + +/** The refusal of `action`: its code, its prepack diagnostics and how many times stdout says each marker. */ +async function refusal(action) { + try { + await action(); + } catch (error) { + const lines = productionFailure(error); + const stdout = typeof error?.result?.stdout === 'string' ? error.result.stdout : ''; + return { + code: /: ([A-Z0-9_]+)$/u.exec(lines.at(-1))?.[1] ?? 'UNEXPECTED', + diagnostics: lines.slice(0, -1), + builds: (stdout.match(/\bESM Build start\b/gu) ?? []).length, + finalized: (stdout.match(/^✓ CLI candidate finalized: /gmu) ?? []).length, + }; + } + return { code: 'ACCEPTED', diagnostics: [], builds: 0, finalized: 0 }; +} + +/** A file's bytes and mode, to restore it exactly. */ +function snapshot(path) { + const bytes = readOrdinaryFile(path, 16 * MIB); + return { path, bytes, sha256: sha256(bytes), mode: lstatSync(path).mode & 0o777 }; +} + +/** Put a snapshot back and prove it is byte for byte the original. */ +function restore(saved) { + rmSync(saved.path, { force: true }); + writeFileSync(saved.path, saved.bytes, { mode: saved.mode, flag: 'wx' }); + chmodSync(saved.path, saved.mode); + assert.equal( + sha256(readOrdinaryFile(saved.path, 16 * MIB)), + saved.sha256, + 'PACKED_CLI_MAP_CONTROL_RESTORE', + ); +} + +/** SHA-256 of every file in a directory, by name; a directory listed as such. */ +function inventory(directory, skip = new Set()) { + return Object.fromEntries( + readdirSync(directory, { withFileTypes: true, recursive: true }) + .map((entry) => join(entry.parentPath ?? entry.path, entry.name)) + .filter((path) => ![...skip].some((name) => path.startsWith(join(directory, name)))) + .sort() + .map((path) => [ + path.slice(directory.length), + lstatSync(path).isFile() ? sha256(readOrdinaryFile(path, 32 * MIB)) : 'directory', + ]), + ); +} + +/** + * Run every control inside the baseline seal. `context` holds the seal's own inputs, its receipt, the + * baseline artifact directory and the emitter's isolated environment and pnpm. + * @returns {Promise<{id: string, expected: string, observed: string}[]>} + */ +async function runControls(context) { + const { candidate, staging, checkout, source, dirty, receipt, artifact, env, pnpm } = context; + const results = []; + const record = (id, expected, observed) => results.push({ id, expected, observed }); + const controls = privateDirectory(join(staging, 'controls')); + let count = 0; + const scratch = (id) => + privateDirectory(join(controls, `${String((count += 1)).padStart(2, '0')}-${id}`)); + const cli = join(candidate, 'apps', 'cli'); + const tarball = join(artifact, receipt.tarball); + const sealed = readPrivateManifest(staging); + const distDigests = () => inventory(join(cli, 'dist')); + + // The chunks the anchors' sources lie in, by their staged maps; the anchor sources' upstream map. + const anchorSources = new Set(ANCHORS.map((anchor) => anchor.source)); + const mapsOf = (name) => + JSON.parse(readOrdinaryFile(join(staging, 'maps', name), 16 * MIB).toString('utf8')); + const chunks = readdirSync(join(staging, 'maps')) + .filter((name) => mapsOf(name).sources.some((path) => anchorSources.has(path))) + .map((name) => name.slice(0, -'.map'.length)) + .sort(); + assert.ok(chunks.length > 0, 'PACKED_CLI_MAP_CONTROL_NO_ANCHOR_CHUNK'); + const plain = sealed.outputs + .map((output) => basename(output.path)) + .filter((name) => !chunks.includes(name)) + .sort(); + + /** A copy of the sealed staging re-recorded for the candidate as it now is, then symbolicated. */ + const resealed = async (id, { forceDirty = dirty, changeMap } = {}) => { + const root = scratch(id); + const copy = privateDirectory(join(root, 'staging')); + privateDirectory(join(copy, 'maps')); + const finalized = JSON.parse( + readOrdinaryFile(join(staging, 'finalized.json'), MIB).toString('utf8'), + ); + finalized.maps = finalized.maps.map((entry) => { + const name = basename(entry.path); + let bytes = readOrdinaryFile(join(staging, 'maps', name), 16 * MIB); + if (changeMap !== undefined && name === `${changeMap.chunk}.map`) { + bytes = Buffer.from(JSON.stringify(changeMap.change(JSON.parse(bytes.toString('utf8'))))); + } + writeFileSync(join(copy, 'maps', name), bytes, { mode: 0o600, flag: 'wx' }); + return { path: entry.path, bytes: bytes.length, sha256: sha256(bytes) }; + }); + finalized.outputs = finalized.outputs.map((entry) => { + const bytes = readOrdinaryFile(join(cli, ...entry.path.split('/')), 8 * MIB); + return { path: entry.path, bytes: bytes.length, sha256: sha256(bytes) }; + }); + writeFileSync(join(copy, 'finalized.json'), JSON.stringify(finalized), { + mode: 0o600, + flag: 'wx', + }); + const digests = new Map( + finalized.outputs.map((output) => [`package/${output.path}`, output.sha256]), + ); + const archive = { + sha256: receipt.tarballSha256, + package: receipt.package, + files: receipt.files.map((file) => + digests.has(file.path) ? { ...file, sha256: digests.get(file.path) } : file, + ), + }; + await sealPrivateManifest({ + candidate, + staging: copy, + checkout, + source, + dirty: forceDirty, + archive, + }); + const work = privateDirectory(join(root, 'work')); + return refusal(() => + symbolicateCandidate({ candidate, staging: copy, checkout, work, archive }), + ); + }; + + /** One pack of the candidate as it now is, into a fresh staging and artifact directory. */ + const pack = async (id, stagingOverride) => { + const root = scratch(id); + const destination = privateDirectory(join(root, 'artifact')); + const fresh = stagingOverride ?? privateDirectory(join(root, 'staging')); + const observed = await refusal(() => + pnpm(['pack', '--pack-destination', destination], { + cwd: cli, + env: { ...env, [CANDIDATE_STAGING_ENV]: fresh }, + }), + ); + return { observed, destination, staging: fresh, root }; + }; + + /** Seal a fresh pack's tarball with fresh work, as the producer would. */ + const seal = (packed) => + refusal(() => + sealPackage(packed.destination, { + repository: candidate, + staging: packed.staging, + source, + dirty, + checkout, + work: privateDirectory(join(packed.root, 'work')), + }), + ); + + // Post-receipt rebuild refusals that need the sealed state come first. + { + // A second prepack into the sealed staging refuses before any build, changing nothing. + const before = { dist: distDigests(), staging: inventory(staging, new Set(['controls'])) }; + const second = await pack('second-prepack', staging); + const unchanged = + JSON.stringify(distDigests()) === JSON.stringify(before.dist) && + JSON.stringify(inventory(staging, new Set(['controls']))) === + JSON.stringify(before.staging) && + readdirSync(second.destination).length === 0 && + second.observed.builds === 0; + record( + 'second-prepack', + 'STAGING_NOT_EMPTY', + unchanged && + second.observed.diagnostics.length === 1 && + second.observed.diagnostics[0] === '✗ CLI candidate finalization refused: STAGING_NOT_EMPTY' + ? 'STAGING_NOT_EMPTY' + : `MARKERS_${second.observed.code}`, + ); + } + { + // A re-seal into the artifact directory, or of the sealed staging beside a copied tarball, refuses + // before any capture runs. + const receiptDigest = sha256( + readOrdinaryFile(join(artifact, 'package-test-receipt.json'), MIB), + ); + const root = scratch('re-seal'); + const works = [privateDirectory(join(root, 'work-1')), privateDirectory(join(root, 'work-2'))]; + const again = await refusal(() => + sealPackage(artifact, { + repository: candidate, + staging, + source, + dirty, + checkout, + work: works[0], + }), + ); + const copied = privateDirectory(join(root, 'artifact')); + cpSync(tarball, join(copied, receipt.tarball)); + const beside = await refusal(() => + sealPackage(copied, { + repository: candidate, + staging, + source, + dirty, + checkout, + work: works[1], + }), + ); + const quiet = + works.every((work) => readdirSync(work).length === 0) && + sha256(readOrdinaryFile(join(artifact, 'package-test-receipt.json'), MIB)) === receiptDigest; + record( + 're-seal-receipt', + 'PACKED_CLI_ALREADY_SEALED', + quiet ? again.code : `MARKERS_${again.code}`, + ); + record( + 're-seal-staging', + 'PACKED_CLI_ALREADY_SEALED', + quiet ? beside.code : `MARKERS_${beside.code}`, + ); + } + + // The re-recording technique alone refuses nothing: the unchanged copy reaches every anchor. + record('re-recorded-baseline', 'ACCEPTED', (await resealed('re-recorded-baseline')).code); + + // Wrong JS by line and by column, a wrong map and permuted sources, on every anchor-bearing chunk. + for (const [index, chunk] of chunks.entries()) { + const path = join(cli, 'dist', chunk); + for (const [kind, change] of [ + ['js-line', shiftLines], + ['js-column', shiftColumns], + ]) { + const saved = snapshot(path); + try { + writeFileSync(path, change(saved.bytes.toString('utf8'))); + record( + `${kind}-${index + 1}`, + 'PACKED_CLI_SYMBOLICATION_ANCHOR_UNMAPPED', + (await resealed(kind)).code, + ); + } finally { + restore(saved); + } + } + record( + `map-line-${index + 1}`, + 'PACKED_CLI_SYMBOLICATION_ANCHOR_UNMAPPED', + ( + await resealed('map-line', { + changeMap: { chunk, change: (map) => ({ ...map, mappings: `;${map.mappings}` }) }, + }) + ).code, + ); + record( + `sources-rotated-${index + 1}`, + 'PACKED_CLI_SYMBOLICATION_ANCHOR_UNMAPPED', + ( + await resealed('sources-rotated', { + changeMap: { chunk, change: (map) => ({ ...map, sources: rotateSources(map.sources) }) }, + }) + ).code, + ); + } + + // A wrong source revision: clean, the Git object refuses the copy; dirty, the anchor positions move. + { + const saved = snapshot(join(candidate, ...ANCHORS[1].source.split('/'))); + try { + writeFileSync(saved.path, `// shifted\n${saved.bytes.toString('utf8')}`); + record( + 'source-clean', + 'PACKED_CLI_SYMBOLICATION_SOURCE_REVISION', + (await resealed('source-clean', { forceDirty: false })).code, + ); + record( + 'source-dirty', + 'PACKED_CLI_SYMBOLICATION_ANCHOR_UNMAPPED', + (await resealed('source-dirty', { forceDirty: true })).code, + ); + } finally { + restore(saved); + } + } + + // A rebuilt dist that differs from the manifest: the sealed state verifies; a changed chunk no frame + // lands in, and a real rebuild from a changed source, refuse. + record( + 'sealed-dist', + 'ACCEPTED', + (await refusal(() => verifySealedDist({ candidate, manifest: sealed, tarball }))).code, + ); + if (plain.length > 0) { + const saved = snapshot(join(cli, 'dist', plain[0])); + try { + writeFileSync(saved.path, Buffer.concat([saved.bytes, Buffer.from(' ')])); + record( + 'dist-plain-chunk', + 'PACKED_CLI_MANIFEST_DIST_DRIFT', + (await refusal(() => verifySealedDist({ candidate, manifest: sealed, tarball }))).code, + ); + } finally { + restore(saved); + } + } + const entry = snapshot(join(cli, 'src', 'index.ts')); + try { + writeFileSync( + entry.path, + `${entry.bytes.toString('utf8')}\nglobalThis.relaviumMapControls = 1;\n`, + ); + const rebuilt = await pack('rebuild'); + const changed = sealed.outputs.some((output) => { + try { + return ( + sha256(readOrdinaryFile(join(cli, ...output.path.split('/')), 8 * MIB)) !== output.sha256 + ); + } catch { + return true; + } + }); + const verified = await refusal(() => + verifySealedDist({ candidate, manifest: sealed, tarball }), + ); + record( + 'dist-rebuilt', + 'PACKED_CLI_MANIFEST_DIST_DRIFT', + rebuilt.observed.code === 'ACCEPTED' && changed + ? verified.code + : `MARKERS_${rebuilt.observed.code}`, + ); + } finally { + restore(entry); + } + + // Upstream maps: a missing, a malformed and a value-truncated map refuse in the finalizer after the + // bundler silently succeeded; a byte-truncated map fails the build; a shifted but valid map is + // accepted there and refused by symbolication. + const upstream = ANCHORS[0].source + .replace(/^packages\/core\/src\//u, 'packages/core/dist/') + .replace(/\.ts$/u, '.js'); + const upstreamMap = join(candidate, ...`${upstream}.map`.split('/')); + const saved = snapshot(upstreamMap); + const original = JSON.parse(saved.bytes.toString('utf8')); + const refusedLine = (code, subject) => + `✗ CLI candidate finalization refused: ${code} (${subject})`; + for (const [id, corrupt, expected] of [ + ['upstream-missing', () => rmSync(upstreamMap), refusedLine('UPSTREAM_MAP_MISSING', upstream)], + [ + 'upstream-shape', + () => writeFileSync(upstreamMap, JSON.stringify({ ...original, sourcesContent: [null] })), + refusedLine('UPSTREAM_MAP_SHAPE', `${upstream}.map`), + ], + [ + 'upstream-index', + () => + writeFileSync( + upstreamMap, + JSON.stringify({ ...original, mappings: `A${vlq(original.sources.length)}AA` }), + ), + refusedLine('UPSTREAM_MAP_MAPPINGS', `${upstream}.map`), + ], + [ + // A map that is no longer JSON fails the bundler's own build, before the finalizer reads it. + 'upstream-cut-bytes', + () => writeFileSync(upstreamMap, saved.bytes.subarray(0, Math.floor(saved.bytes.length / 2))), + '✗ CLI build failed', + ], + [ + 'upstream-cut-value', + () => + writeFileSync( + upstreamMap, + JSON.stringify({ ...original, mappings: cutInsideValue(original.mappings) }), + ), + refusedLine('UPSTREAM_MAP_MAPPINGS', `${upstream}.map`), + ], + ]) { + try { + corrupt(); + const packed = await pack(id); + // The bundler silently succeeded once, nothing was finalized, staged or packed, and the one + // prepack diagnostic is the expected fixed line. + const markers = [ + packed.observed.builds, + packed.observed.finalized, + readdirSync(packed.staging).length, + readdirSync(packed.destination).length, + packed.observed.diagnostics.length, + ]; + const line = packed.observed.diagnostics.at(-1); + record( + id, + expected, + markers.join() === '1,0,0,0,1' && line === expected + ? expected + : `MARKERS(${markers.join()}) ${line ?? packed.observed.code}`, + ); + } finally { + restore(saved); + } + } + try { + const mappings = shiftOriginalColumns(original.mappings, { + sources: original.sources.length, + names: original.names.length, + }); + // The shift is strictly valid: the finalizer's own checks admit it. + const lines = decodeMappings(mappings, { + sources: original.sources.length, + names: original.names.length, + }); + assertGeneratedRange( + lines, + lineLengths( + readOrdinaryFile(join(candidate, ...upstream.split('/')), 16 * MIB).toString('utf8'), + ), + ); + assertOriginalRange( + lines, + original.sources.map((path) => + lineLengths( + readOrdinaryFile(join(dirname(upstreamMap), ...path.split('/')), 16 * MIB).toString( + 'utf8', + ), + ), + ), + ); + writeFileSync(upstreamMap, JSON.stringify({ ...original, mappings })); + const packed = await pack('upstream-shifted'); + const accepted = packed.observed.code === 'ACCEPTED'; + record( + 'upstream-shifted', + 'PACKED_CLI_SYMBOLICATION_ANCHOR_UNMAPPED', + accepted ? (await seal(packed)).code : `MARKERS_${packed.observed.code}`, + ); + } finally { + restore(saved); + } + + // Restoration: every corrupted file is back, so a fresh pack seals again. + const restored = await pack('restoration'); + record( + 'restoration', + 'ACCEPTED', + restored.observed.code === 'ACCEPTED' + ? (await seal(restored)).code + : `MARKERS_${restored.observed.code}`, + ); + return results; +} + +/** + * Run the controls through one real production of `repository` and return their results with the + * source identity. The developer checkout must be the same before and after. + * @param {{repository: string, environment: Record, + * temporary?: string}} options + */ +export async function runMapControls({ repository, environment, temporary = tmpdir() }) { + const before = await sourceIdentity(repository); + const root = realpathSync.native(mkdtempSync(join(temporary, 'relavium-map-controls-'))); + chmodSync(root, 0o700); + let results = []; + let emitted; + try { + const produced = await producePackage({ + repository, + destination: join(root, 'artifact'), + environment, + temporary, + ports: { + emit: (options) => { + emitted = { env: options.env, pnpm: options.pnpm }; + return emitFreshPackage(options); + }, + seal: async (artifact, inputs) => { + const receipt = await sealPackage(artifact, inputs); + results = await runControls({ + candidate: inputs.repository, + staging: inputs.staging, + checkout: inputs.checkout, + source: inputs.source, + dirty: inputs.dirty, + receipt, + artifact, + ...emitted, + }); + return receipt; + }, + }, + }); + const after = await sourceIdentity(repository); + assert.deepEqual( + [after.source, after.status, after.unchecked], + [before.source, before.status, before.unchecked], + 'PACKED_CLI_MAP_CONTROL_CHECKOUT_CHANGED', + ); + return { source: produced.receipt.source, dirty: produced.identity.dirty, results }; + } finally { + rmSync(root, { recursive: true, force: true }); + } +} + +if ( + process.argv[1] !== undefined && + realpathSync.native(process.argv[1]) === fileURLToPath(import.meta.url) +) { + const started = Date.now(); + let outcome; + try { + outcome = await runMapControls({ + repository: fileURLToPath(new URL('../../', import.meta.url)), + environment: process.env, + }); + } catch (error) { + for (const line of productionFailure(error)) console.error(line); + process.exit(1); + } + const failed = outcome.results.filter((result) => result.observed !== result.expected); + console.log( + JSON.stringify({ + scope: 'product copies of one real package-test production', + source: outcome.source, + dirty: outcome.dirty, + controls: outcome.results.length, + failed: failed.map((result) => result.id), + results: outcome.results, + seconds: Math.round((Date.now() - started) / 1000), + }), + ); + if (failed.length > 0) { + console.error( + `✗ CLI product map controls failed: ${failed.map((result) => result.id).join(', ')}`, + ); + process.exit(1); + } +} diff --git a/tools/cli-package/map-controls.test.mjs b/tools/cli-package/map-controls.test.mjs new file mode 100644 index 00000000..1666fb1e --- /dev/null +++ b/tools/cli-package/map-controls.test.mjs @@ -0,0 +1,130 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { + cutInsideValue, + encodeMappings, + rotateSources, + shiftColumns, + shiftLines, + shiftOriginalColumns, +} from './map-controls.mjs'; +import { SourceMapError, decodeMappings } from './source-map.mjs'; + +// Importing the runner runs no control: only its command-line entry point does. +const COUNTS = { sources: 3, names: 2 }; +const refuses = (code) => (error) => error instanceof Error && error.message === code; + +/** Deterministic mappings with empty lines, unmapped and named segments and backward moves. */ +function sample() { + let seed = 11; + const next = (bound) => { + seed = (seed * 1103515245 + 12345) % 2 ** 31; + // The high bits: an LCG's low bits cycle and would never draw some kinds. + return Math.floor(seed / 65536) % bound; + }; + const lines = []; + for (let line = 0; line < 30; line += 1) { + const segments = []; + let column = next(3); + for (let index = next(12); index > 0; index -= 1) { + column += 1 + next(400); + const kind = next(4); + if (kind === 0) segments.push({ generatedColumn: column }); + else { + const segment = { + generatedColumn: column, + source: next(3), + originalLine: next(200), + originalColumn: kind === 1 ? 0 : next(120), + }; + if (kind === 3) segment.name = next(2); + segments.push(segment); + } + } + lines.push(segments); + } + // Every kind of segment the encoder must write is present. + const all = lines.flat(); + assert.ok(all.some((segment) => segment.source === undefined)); + assert.ok(all.some((segment) => segment.name !== undefined)); + assert.ok(all.some((segment) => segment.originalColumn > 0)); + assert.ok(all.some((segment) => segment.originalColumn === 0)); + assert.ok(lines.some((segments) => segments.length === 0)); + return lines; +} + +test('the encoder writes back exactly what the strict decoder reads', () => { + const lines = sample(); + const mappings = encodeMappings(lines); + assert.deepEqual(decodeMappings(mappings, COUNTS), lines); + // Canonical input re-encodes to the same text, empty lines included. + assert.equal(encodeMappings(decodeMappings(mappings, COUNTS)), mappings); + assert.equal(encodeMappings(decodeMappings(';;AAAA;', COUNTS)), ';;AAAA;'); +}); + +test('a shifted map moves every original column after a line start one left, and nothing else', () => { + const lines = sample(); + const shifted = decodeMappings(shiftOriginalColumns(encodeMappings(lines), COUNTS), COUNTS); + assert.equal(shifted.length, lines.length); + lines.forEach((segments, line) => + segments.forEach((segment, index) => { + const moved = shifted[line][index]; + assert.equal(moved.generatedColumn, segment.generatedColumn); + assert.equal(moved.source, segment.source); + assert.equal(moved.originalLine, segment.originalLine); + assert.equal(moved.name, segment.name); + assert.equal( + moved.originalColumn, + segment.originalColumn === undefined || segment.originalColumn === 0 + ? segment.originalColumn + : segment.originalColumn - 1, + ); + }), + ); + // A map with nothing to shift refuses rather than leaving the control hollow. + assert.throws( + () => shiftOriginalColumns('AAAA;CAAA', COUNTS), + refuses('PACKED_CLI_MAP_CONTROL_NOTHING_SHIFTED'), + ); +}); + +test('rotated sources leave no source at its index', () => { + for (const sources of [ + ['a.ts', 'b.ts'], + ['a.ts', 'b.ts', 'c.ts'], + ['a.ts', 'b.ts', 'c.ts', 'd.ts', 'e.ts'], + ]) { + const rotated = rotateSources(sources); + assert.deepEqual([...rotated].sort(), [...sources].sort()); + assert.ok(rotated.every((source, index) => source !== sources[index])); + } + for (const sources of [[], ['a.ts'], ['a.ts', 'a.ts']]) { + assert.throws(() => rotateSources(sources), refuses('PACKED_CLI_MAP_CONTROL_SOURCES')); + } +}); + +test('the JS moves one line or one column, keeping a hashbang on the first line', () => { + assert.equal(shiftLines('a();\nb();\n'), '\na();\nb();\n'); + assert.equal(shiftLines('#!/usr/bin/env node\na();\n'), '#!/usr/bin/env node\n\na();\n'); + assert.throws( + () => shiftLines('#!/usr/bin/env node'), + refuses('PACKED_CLI_MAP_CONTROL_HASHBANG'), + ); + assert.equal(shiftColumns('a();\nb();\n'), ' a();\n b();\n '); + assert.equal(shiftColumns('#!/usr/bin/env node\na();'), '#!/usr/bin/env node\n a();'); +}); + +test('a value cut after its first continuation digit never ends', () => { + const mappings = encodeMappings(sample()); + const cut = cutInsideValue(mappings); + assert.ok(mappings.startsWith(cut)); + assert.match(cut.at(-1), /[g-z0-9+/]/u); + assert.throws( + () => decodeMappings(cut, COUNTS), + (error) => error instanceof SourceMapError && error.code === 'MAPPINGS_TRUNCATED', + ); + assert.throws( + () => cutInsideValue('AAAA;CAAA'), + refuses('PACKED_CLI_MAP_CONTROL_NO_CONTINUATION'), + ); +}); From 5e914cc3e4be55abb9cbca6d7a2bb7fab562e719 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sat, 24 Oct 2026 18:24:56 +0300 Subject: [PATCH 46/97] ci(repo): require the product map controls before ci Step 9a.2b, third commit. A new map-controls job runs pnpm check:map-controls as a second, independent real production at the commit, shaped like pack-cli: ubuntu, at most 20 minutes, contents read, the remote-cache credentials cleared, setup-node without a pnpm cache, a checkout without persisted credentials, no build, cache, upload or download step, and the package source identity set as pack-cli sets it. Its private maps, manifests and copies stay in the producer's owner-only root. - ci needs it, and a guard refuses every result except success; the packed-cli aggregate does not, and the required contexts stay ci and coverage. release.yml gets no such job (9d owns that graph). - pnpm run ci runs check:map-controls right after smoke:cli-package. - require-packed-cli.test.mjs pins the job's exact shape and refuses a pnpm cache, a cache action, an upload, persisted credentials, a build, a tolerated failure, a disabled job, a longer deadline, a missing source identity, removal from ci's needs, joining the aggregate and a removed, disabled or permissive guard (each must fail an assertion, not a TypeError); require-w8-acceptance.test.mjs pins ci's needs, the script and its place in the chain. Refs: ADR-0111, #46 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 39 +++++++- package.json | 2 +- tools/ci/require-packed-cli.test.mjs | 115 +++++++++++++++++++++++- tools/ci/require-w8-acceptance.test.mjs | 12 +++ 4 files changed, 163 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 086c8222..846a5a62 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ # Turborepo cache, plus the formatting check and the no-vendor-type-across-the-seam fence. # # Branch protection (set in the GitHub repo settings, not here): `ci` and `coverage` are the REQUIRED -# status checks to merge into `main`. Wire, usage, filesystem, secrets and packed CLI jobs feed `ci` through an explicit +# status checks to merge into `main`. Wire, usage, filesystem, secrets, packed CLI and product map control jobs feed `ci` through an explicit # success-only aggregation; they cannot silently skip or fail while `ci` stays green. The other jobs # self-label `(advisory)` — `floor-check`, `peer-dep-gate` and `windows-concurrency`. # @@ -57,7 +57,8 @@ env: jobs: ci: - needs: [wire-mutations, usage-mutations, fs-mutations, secrets-mutations, packed-cli] + needs: + [wire-mutations, usage-mutations, fs-mutations, secrets-mutations, packed-cli, map-controls] if: ${{ always() }} name: lint · typecheck · test runs-on: ubuntu-latest @@ -102,6 +103,11 @@ jobs: PACKED_CLI_RESULT: ${{ needs.packed-cli.result }} run: node -e "require('node:assert/strict').equal(process.env.PACKED_CLI_RESULT, 'success')" + - name: Require complete product map controls + env: + MAP_CONTROLS_RESULT: ${{ needs.map-controls.result }} + run: node -e "require('node:assert/strict').equal(process.env.MAP_CONTROLS_RESULT, 'success')" + # A squash/main-based checkout need not contain development's immutable predecessor object. # Fetch only this canonical pin during CI setup; the acceptance harness itself stays offline. - name: Ensure immutable replay predecessor is available @@ -653,6 +659,35 @@ jobs: retention-days: 7 if-no-files-found: error + # Product map controls (Wave 3 Step 9a.2b): a second, independent real production at this commit whose + # every named corruption must refuse with its exact code; its private maps, manifests and copies stay in + # the producer's owner-only root and are removed with it. Uncached, and nothing is uploaded. + map-controls: + name: product map controls + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + env: + TURBO_TOKEN: '' + TURBO_TEAM: '' + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + - name: Set up pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 + - name: Set up Node + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version-file: .nvmrc + - name: Install (frozen lockfile) + run: pnpm install --frozen-lockfile + - name: Corrupt product copies and require each exact refusal + env: + RELAVIUM_PACKAGE_SOURCE_SHA: ${{ github.sha }} + run: pnpm check:map-controls + packed-cli-consumer: name: installed CLI (${{ matrix.id }}) needs: pack-cli diff --git a/package.json b/package.json index 8cb50213..10ca2fb2 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "coverage": "vitest run --coverage", "coverage:enforced": "node tools/coverage-gate/run.mjs", "ci:workspaces": "turbo run lint typecheck && turbo run test --concurrency=1", - "ci": "pnpm lint:test-isolation && pnpm ci:workspaces && pnpm db:sync-check && pnpm typecheck:tools && pnpm lint:tools && turbo run build format:check && pnpm lint:fence-check && pnpm lint:engine-deps && pnpm lint:bundle-closure && pnpm smoke:cli-package && pnpm smoke:cli-bootstrap && pnpm smoke:cli && pnpm smoke:media-packaging && pnpm smoke:overflow-capture && pnpm smoke:budget-replay && pnpm test:acceptance-tools && pnpm check:wire-mutations && pnpm check:usage-mutations && pnpm check:fs-mutations && pnpm check:secrets-mutations", + "ci": "pnpm lint:test-isolation && pnpm ci:workspaces && pnpm db:sync-check && pnpm typecheck:tools && pnpm lint:tools && turbo run build format:check && pnpm lint:fence-check && pnpm lint:engine-deps && pnpm lint:bundle-closure && pnpm smoke:cli-package && pnpm check:map-controls && pnpm smoke:cli-bootstrap && pnpm smoke:cli && pnpm smoke:media-packaging && pnpm smoke:overflow-capture && pnpm smoke:budget-replay && pnpm test:acceptance-tools && pnpm check:wire-mutations && pnpm check:usage-mutations && pnpm check:fs-mutations && pnpm check:secrets-mutations", "lint:test-isolation": "node tools/test-isolation/check.mjs && node tools/test-isolation/check-ownership.mjs", "lint:fence-check": "node tools/lint-fixtures/assert-fence.mjs", "lint:engine-deps": "node tools/engine-deps/check.mjs", diff --git a/tools/ci/require-packed-cli.test.mjs b/tools/ci/require-packed-cli.test.mjs index d9e22611..64c4e9dc 100644 --- a/tools/ci/require-packed-cli.test.mjs +++ b/tools/ci/require-packed-cli.test.mjs @@ -126,7 +126,7 @@ function checkAggregate(job, packId, consumerId) { } function checkUnprivilegedPackageJobs(workflow) { - for (const id of ['pack-cli', 'packed-cli-consumer', 'packed-cli']) { + for (const id of ['pack-cli', 'packed-cli-consumer', 'packed-cli', 'map-controls']) { const job = workflow.jobs[id]; for (const key of ['TURBO_TOKEN', 'TURBO_TEAM']) { assert.equal(job.env?.[key], '', `${id} must clear inherited ${key}`); @@ -141,7 +141,7 @@ test('all PR-controlled installed-package jobs clear inherited remote-cache cred checkUnprivilegedPackageJobs(readWorkflow('ci')); }); -for (const id of ['pack-cli', 'packed-cli-consumer', 'packed-cli']) +for (const id of ['pack-cli', 'packed-cli-consumer', 'packed-cli', 'map-controls']) for (const key of ['TURBO_TOKEN', 'TURBO_TEAM']) for (const value of [undefined, '${{ secrets.REMOTE_CACHE_CREDENTIAL }}']) test(`package privilege guard refuses ${id} ${key}: ${String(value)}`, () => { @@ -227,6 +227,117 @@ for (const [name, packId, consumerId] of [ }); } +const mapControlsCommand = 'pnpm check:map-controls'; +const mapControlsGuard = 'Require complete product map controls'; + +/** The product map controls job: uncached, unprivileged, no upload, joined to required CI only. */ +function checkMapControls(workflow) { + const job = workflow.jobs['map-controls']; + requiredJob(job); + assert.equal(job.needs, undefined); + assert.equal(job.strategy, undefined); + assert.equal(job['runs-on'], 'ubuntu-latest'); + // Checkout, pnpm and Node only: no cache, upload or download action. + assert.deepEqual( + job.steps.filter((step) => step.uses).map((step) => step.uses), + [actions.checkout, actions.pnpm, actions.node], + ); + assert.deepEqual(job.steps.find((step) => step.uses === actions.checkout).with, { + 'persist-credentials': false, + }); + assert.equal(job.steps.find((step) => step.uses === actions.pnpm).with, undefined); + assert.deepEqual(job.steps.find((step) => step.uses === actions.node).with, { + 'node-version-file': '.nvmrc', + }); + assert.deepEqual( + job.steps.filter((step) => step.run).map((step) => step.run), + ['pnpm install --frozen-lockfile', mapControlsCommand], + ); + assert.deepEqual(job.steps.find((step) => step.run === mapControlsCommand).env, { + RELAVIUM_PACKAGE_SOURCE_SHA: '${{ github.sha }}', + }); + // Required CI waits for it and refuses every result except success; the package aggregate does not. + const ci = workflow.jobs.ci; + assert.ok(ci.needs.includes('map-controls')); + assert.ok(![workflow.jobs['packed-cli'].needs].flat().includes('map-controls')); + const guard = ci.steps.find((step) => step.name === mapControlsGuard); + assert.ok(guard, mapControlsGuard); + assert.deepEqual(guard.env, { MAP_CONTROLS_RESULT: '${{ needs.map-controls.result }}' }); + assert.equal( + guard.run, + "node -e \"require('node:assert/strict').equal(process.env.MAP_CONTROLS_RESULT, 'success')\"", + ); + assert.equal(guard.if, undefined); + assert.equal(guard['continue-on-error'], undefined); +} + +test('required CI runs the product map controls uncached and without uploads; release does not', () => { + checkMapControls(readWorkflow('ci')); + // The release workflow's graph belongs to Step 9d (the 2026-10-24 ruling). + assert.equal(readWorkflow('release').jobs['map-controls'], undefined); +}); + +for (const [name, mutate] of [ + [ + 'a pnpm cache', + (job) => (job.steps.find((step) => step.uses === actions.node).with.cache = 'pnpm'), + ], + [ + 'a cache action', + (job) => + job.steps.splice(3, 0, { + uses: 'actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830', + with: { path: '.turbo', key: 'turbo' }, + }), + ], + [ + 'an upload', + (job) => job.steps.push({ uses: actions.upload, with: { name: 'maps', path: 'x' } }), + ], + ['credentials kept in the checkout', (job) => (job.steps[0].with['persist-credentials'] = true)], + ['a build before the controls', (job) => job.steps.splice(4, 0, { run: 'pnpm turbo run build' })], + ['a tolerated failure', (job) => (job['continue-on-error'] = true)], + ['a disabled job', (job) => (job.if = '${{ false }}')], + ['a longer deadline', (job) => (job['timeout-minutes'] = 30)], + [ + 'no source identity', + (job) => delete job.steps.find((step) => step.run === mapControlsCommand).env, + ], +]) + test(`map controls job refuses ${name}`, () => { + const workflow = readWorkflow('ci'); + mutate(workflow.jobs['map-controls']); + assert.throws(() => checkMapControls(workflow), assert.AssertionError); + }); + +for (const [name, mutate] of [ + [ + 'removal from required CI', + (jobs) => (jobs.ci.needs = jobs.ci.needs.filter((id) => id !== 'map-controls')), + ], + [ + 'joining the package aggregate', + (jobs) => (jobs['packed-cli'].needs = [...jobs['packed-cli'].needs, 'map-controls']), + ], + [ + 'a removed guard', + (jobs) => (jobs.ci.steps = jobs.ci.steps.filter((step) => step.name !== mapControlsGuard)), + ], + [ + 'a disabled guard', + (jobs) => (jobs.ci.steps.find((step) => step.name === mapControlsGuard).if = '${{ false }}'), + ], + [ + 'a guard that admits other results', + (jobs) => (jobs.ci.steps.find((step) => step.name === mapControlsGuard).run = 'true'), + ], +]) + test(`required CI refuses ${name} of the map controls`, () => { + const workflow = readWorkflow('ci'); + mutate(workflow.jobs); + assert.throws(() => checkMapControls(workflow), assert.AssertionError); + }); + test('required CI joins packed acceptance and explicitly refuses every result except success', () => { const job = readWorkflow('ci').jobs.ci; assert.ok(job.needs.includes('packed-cli')); diff --git a/tools/ci/require-w8-acceptance.test.mjs b/tools/ci/require-w8-acceptance.test.mjs index d33e642e..de525fd2 100644 --- a/tools/ci/require-w8-acceptance.test.mjs +++ b/tools/ci/require-w8-acceptance.test.mjs @@ -42,6 +42,7 @@ test('the actual required workflow joins both full offline controls and checks t 'fs-mutations', 'secrets-mutations', 'packed-cli', + 'map-controls', ]); const packedGuard = ci.steps.find((step) => step.name === 'Require complete packed CLI controls'); assert.equal(packedGuard.env.PACKED_CLI_RESULT, '${{ needs.packed-cli.result }}'); @@ -132,4 +133,15 @@ test('the actual required workflow joins both full offline controls and checks t ); for (const kind of ['wire', 'usage', 'fs', 'secrets']) assert.ok(packageJson.scripts.ci.split(' && ').includes(`pnpm check:${kind}-mutations`)); + // The product map controls run locally right after the producer and consumer smoke. + assert.equal( + packageJson.scripts['check:map-controls'], + 'node tools/cli-package/map-controls.mjs', + ); + const chain = packageJson.scripts.ci.split(' && '); + assert.equal( + chain.indexOf('pnpm check:map-controls'), + chain.indexOf('pnpm smoke:cli-package') + 1, + ); + assert.equal(chain.filter((step) => step === 'pnpm check:map-controls').length, 1); }); From c3a41a77856b27a0c8c15cb329fd3f7f6db180fa Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sat, 24 Oct 2026 18:27:18 +0300 Subject: [PATCH 47/97] feat(repo): flip the package-test receipt to package_test Step 9a.2b, fourth commit, after the controls that back it: the receipt mode becomes package_test, as the 9a.2 rulings fixed, with version 1 and its twelve fields kept. - binding.mjs exports one PACKAGE_TEST_MODE, used by the seal and by both receipt validators, so a half-flip cannot exist; the earlier compiled_package_test is refused, and a test pins that refusal. Consumers and the aggregate inherit the flip through the shared validator and change no schema. - seal, produce and release-guard tests follow the new mode. - The ci.yml and release.yml pack-step comments say what backs the mode: the required map-controls job at the same commit. The mode is a claim, not evidence of the map checks; nothing reads it as proof. Gates: the binding, seal, produce, release-guard and workflow pin tests, and smoke:cli-package with the real producer and consumers. Refs: ADR-0111, #46 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 3 ++- .github/workflows/release.yml | 3 ++- tools/cli-package/binding.mjs | 11 +++++++++-- tools/cli-package/binding.test.mjs | 4 +++- tools/cli-package/produce.test.mjs | 10 +++++----- tools/cli-package/release-guard.test.mjs | 2 +- tools/cli-package/seal.mjs | 4 ++-- tools/cli-package/seal.test.mjs | 2 +- 8 files changed, 25 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 846a5a62..5ae00c15 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -644,7 +644,8 @@ jobs: # The runner builds inlined dependencies fresh in an owned candidate tree, then prepack performs the sole # final CLI build, where original-TS maps are built, validated, staged privately and discarded (Wave 3 # Step 9a.1); the seal then writes the private manifest and symbolicates actual packed frames, both - # discarded too (Step 9a.2a); only the tarball and its receipt leave this job. + # discarded too (Step 9a.2a); only the tarball and its package_test receipt leave this job. The map-controls + # job below supplies the negative controls behind that mode (Step 9a.2b). - name: Pack once and independently inspect the public package env: RELAVIUM_PACKAGE_SOURCE_SHA: ${{ github.sha }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cbbb9d92..9edcb539 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -102,7 +102,8 @@ jobs: # The same required package runner owns fresh dependencies and the sole prepack CLI build in an owned # candidate tree, where original-TS maps are built, validated, staged privately and discarded (Wave 3 # Step 9a.1), with the private manifest and the packed-frame symbolication (Step 9a.2a); their - # encrypted release custody is Steps 9b–9d. + # encrypted release custody is Steps 9b–9d. The receipt records package_test; the product map controls + # behind that mode run in the required CI workflow at the same commit (Step 9a.2b), not in this one. - name: Pack once and independently inspect the public package env: RELAVIUM_PACKAGE_SOURCE_SHA: ${{ github.sha }} diff --git a/tools/cli-package/binding.mjs b/tools/cli-package/binding.mjs index 93713b8d..b0c7dc2a 100644 --- a/tools/cli-package/binding.mjs +++ b/tools/cli-package/binding.mjs @@ -7,6 +7,13 @@ import { REQUIRED_SMOKE_CASES } from './smoke/cases.mjs'; /** @typedef {{path: string, bytes: number, sha256: string, mode: number}} PublicFile */ /** @typedef {{source: string, origin: Origin, nodeMajor: number, allowLocal?: boolean}} Expected */ +/** + * The package-test receipt's mode (Step 9a.2b): the producer verified the maps and symbolicated actual + * packed frames before writing it, and the product map controls are required by the same commit's + * `ci`. The mode is a claim, not evidence of those checks; the earlier `compiled_package_test` is refused. + */ +export const PACKAGE_TEST_MODE = 'package_test'; + export const REQUIRED_CONSUMER_LEGS = Object.freeze([ 'current-linux', 'current-macos', @@ -245,7 +252,7 @@ export function validatePackageReceipt(value, expected) { ]); if ( receipt.version !== 1 || - receipt.mode !== 'compiled_package_test' || + receipt.mode !== PACKAGE_TEST_MODE || receipt.source !== expected.source || typeof receipt.dirty !== 'boolean' ) @@ -268,7 +275,7 @@ export function validatePackageReceipt(value, expected) { if (manifest.version !== packageVersion) refuse('PACKAGE'); return { version: 1, - mode: 'compiled_package_test', + mode: PACKAGE_TEST_MODE, source: expected.source, dirty: receipt.dirty, packageVersion, diff --git a/tools/cli-package/binding.test.mjs b/tools/cli-package/binding.test.mjs index 804f615c..dec18225 100644 --- a/tools/cli-package/binding.test.mjs +++ b/tools/cli-package/binding.test.mjs @@ -25,7 +25,7 @@ const expected = { source, origin: { kind: 'local' }, nodeMajor: 24 }; function receipt() { return { version: 1, - mode: 'compiled_package_test', + mode: 'package_test', source, dirty: false, packageVersion: '0.1.1', @@ -122,6 +122,8 @@ test('strict receipt refuses omitted or added private/custody claims', () => { } for (const changed of [ { mode: 'release' }, + // The mode before the product map controls existed. + { mode: 'compiled_package_test' }, { source: '0'.repeat(40) }, { dirty: 'false' }, { version: 2 }, diff --git a/tools/cli-package/produce.test.mjs b/tools/cli-package/produce.test.mjs index 325b7dfc..b7ebe443 100644 --- a/tools/cli-package/produce.test.mjs +++ b/tools/cli-package/produce.test.mjs @@ -86,7 +86,7 @@ function ports({ plantCache = false, failEmit = false, dirty = false } = {}) { seal: async (artifactDirectory, candidate) => { seen.seal = { artifactDirectory, ...candidate }; seen.workMode = statSync(candidate.work).mode & 0o777; - return { mode: 'compiled_package_test', source: candidate.source, files: [] }; + return { mode: 'package_test', source: candidate.source, files: [] }; }, }, }; @@ -282,7 +282,7 @@ test( return seal(destination); }, noBuildCache: () => {}, - seal: () => ({ mode: 'compiled_package_test' }), + seal: () => ({ mode: 'package_test' }), }, }); } finally { @@ -843,7 +843,7 @@ await producePackage({ noBuildCache: () => {}, seal: () => { if (${JSON.stringify(during)} === 'seal') interrupt(); - return { mode: 'compiled_package_test' }; + return { mode: 'package_test' }; }, remove: (path) => { if (${JSON.stringify(during)} === 'removal' && path.includes('relavium-package-candidate-')) @@ -991,7 +991,7 @@ test('the producer entry point hands the checkout, the artifact directory and it `import { writeFileSync } from 'node:fs';\n` + `export async function producePackage({ repository, destination, environment }) {\n` + ` writeFileSync(process.env.RELAVIUM_ENTRY_MARKER, JSON.stringify({ repository, destination, marker: environment.RELAVIUM_ENTRY_MARKER, same: environment === process.env }));\n` + - ` return { receipt: { mode: 'compiled_package_test', source: 's', tarballSha256: 'h', files: [1, 2] }, identity: { files: 3 } };\n` + + ` return { receipt: { mode: 'package_test', source: 's', tarballSha256: 'h', files: [1, 2] }, identity: { files: 3 } };\n` + `}\n`, ); const cwd = fresh('entry-cwd'); @@ -1013,7 +1013,7 @@ test('the producer entry point hands the checkout, the artifact directory and it same: true, }); assert.deepEqual(JSON.parse(result.stdout), { - mode: 'compiled_package_test', + mode: 'package_test', source: 's', sha256: 'h', files: 2, diff --git a/tools/cli-package/release-guard.test.mjs b/tools/cli-package/release-guard.test.mjs index ea74df17..6b3c5e00 100644 --- a/tools/cli-package/release-guard.test.mjs +++ b/tools/cli-package/release-guard.test.mjs @@ -36,7 +36,7 @@ function runGuard(script, args, cwd = scratch) { const PACKAGE_TEST = { 'relavium-0.1.1.tgz': 'tarball', - 'package-test-receipt.json': JSON.stringify({ mode: 'compiled_package_test' }), + 'package-test-receipt.json': JSON.stringify({ mode: 'package_test' }), }; for (const [label, files, code] of [ diff --git a/tools/cli-package/seal.mjs b/tools/cli-package/seal.mjs index 1d37a0c2..ee6b8730 100644 --- a/tools/cli-package/seal.mjs +++ b/tools/cli-package/seal.mjs @@ -9,7 +9,7 @@ import { createRequire } from 'node:module'; import { lstatSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; import { basename, isAbsolute, join } from 'node:path'; import { inspectPackageArchive } from './archive.mjs'; -import { originFromEnvironment, validatePackageReceipt } from './binding.mjs'; +import { PACKAGE_TEST_MODE, originFromEnvironment, validatePackageReceipt } from './binding.mjs'; import { flatFiles, oneTarball, readOrdinaryFile, sha256 } from './files.mjs'; import { MANIFEST_FILE, sealPrivateManifest, verifySealedDist } from './manifest.mjs'; import { symbolicateCandidate } from './symbolicate.mjs'; @@ -162,7 +162,7 @@ export async function sealPackage( await verify({ candidate: repository, manifest: sealed, tarball }); const receipt = { version: 1, - mode: 'compiled_package_test', + mode: PACKAGE_TEST_MODE, source, dirty, packageVersion: archive.package.version, diff --git a/tools/cli-package/seal.test.mjs b/tools/cli-package/seal.test.mjs index 9483ff6b..026141c9 100644 --- a/tools/cli-package/seal.test.mjs +++ b/tools/cli-package/seal.test.mjs @@ -395,7 +395,7 @@ test('the seal binds a locally produced candidate it is given, never the reposit const receipt = await withOrigin({}, () => sealWith(artifact, { repository: root, staging, source: SOURCE, dirty: true }), ); - assert.equal(receipt.mode, 'compiled_package_test'); + assert.equal(receipt.mode, 'package_test'); assert.equal(receipt.source, SOURCE); assert.equal(receipt.dirty, true); assert.equal(receipt.packageVersion, '0.1.1'); From 77ee5d0a43c967e8070aab005487867f12601466 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sat, 24 Oct 2026 18:29:13 +0300 Subject: [PATCH 48/97] docs(repo): record the Step 9a.2b implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 9a.2b's documents, after its four code commits: - packaging.md: the Status line; the shared receipt now package_test; the 9a.1 and 9a.2a notes on the earlier mode; the package-test receipt's full field list; and a new section, "Product map controls and the package-test receipt — Step 9a.2b": the rebuild refusals, the controls and their exact codes, the runner's output, its place in pnpm run ci and the required map-controls job, why that job's second production is not a candidate retry, the mode as a claim rather than evidence, release.yml's dry-run backed by the same commit's ci, and the limits. - tools/cli-package/README.md: the new mode, the refusals and the runner. - The phase register's 9a.2b row and implementation checkpoint, and the status lines in CLAUDE.md, AGENTS.md, current.md, the CLI reference and deferred-tasks.md. Refs: ADR-0111, #46 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 +- CLAUDE.md | 4 +- docs/reference/cli/README.md | 2 +- docs/reference/cli/packaging.md | 84 ++++++++++++++++--- docs/roadmap/current.md | 6 +- docs/roadmap/deferred-tasks.md | 2 +- .../phase-2.5.5-hardening-and-remediation.md | 23 ++++- tools/cli-package/README.md | 10 ++- 8 files changed, 111 insertions(+), 24 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1d4923d9..03b09cb2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,7 +117,7 @@ explicit; source CI does not certify later documentation commits. The joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now -accepted on `development`, below, as is Step 9a.2a; Steps 9a.2b–19 have not started) and whole-wave acceptance remains Step 19. +accepted on `development`, below, as is Step 9a.2a; Step 9a.2b is implemented and under review; Steps 9b–19 have not started) and whole-wave acceptance remains Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification @@ -130,7 +130,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b is next. `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-implementation-checkpoint--2026-10-24)). `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. See [docs/roadmap/current.md](docs/roadmap/current.md) for live status. diff --git a/CLAUDE.md b/CLAUDE.md index f088e339..f648f233 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -206,7 +206,7 @@ explicit; source CI does not certify later documentation commits. The joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now -accepted on `development`, below, as is Step 9a.2a; Steps 9a.2b–19 have not started) and whole-wave acceptance remains Step 19. +accepted on `development`, below, as is Step 9a.2a; Step 9a.2b is implemented and under review; Steps 9b–19 have not started) and whole-wave acceptance remains Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification @@ -219,7 +219,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b is next. `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-implementation-checkpoint--2026-10-24)). `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. For live status, per-PR history, milestone dates, and open obligations, see the canonical diff --git a/docs/reference/cli/README.md b/docs/reference/cli/README.md index 751cc665..8c1ea913 100644 --- a/docs/reference/cli/README.md +++ b/docs/reference/cli/README.md @@ -13,5 +13,5 @@ Part of [reference/](../README.md). | [commands.md](commands.md) | The full `relavium` command surface (run, chat, chat-resume/list/export, agent run, list, create, import, export, logs, gate, init, provider) plus interactive TUI vs CI JSON mode. | | [home.md](home.md) | The bare-invocation interactive **Home** — the TTY gate, the read-only management strip over `history.db`, the Home↔chat mode machine + keymap, bracketed paste, the min-terminal degrade, and the signal/exit-code lifecycle (2.5.B / ADR-0054). | | [chat-session.md](chat-session.md) | The `relavium chat` agent-session REPL — entry, agent/model selection, the multi-turn loop, streaming, tool availability, the `--json` session-event stream, and exit code 4. | -| [packaging.md](packaging.md) | Wave 3 finite runtime graph, installed-package consumers and Step 8 native activation accepted on `development`; the Step 9a.1 candidate build and finalizer are accepted on `development` and Step 9a.2a's private manifest and symbolication are accepted on `development`; the remaining Step 9 map/custody and release gates remain staged. | +| [packaging.md](packaging.md) | Wave 3 finite runtime graph, installed-package consumers and Step 8 native activation accepted on `development`; the Step 9a.1 candidate build and finalizer are accepted on `development` and Step 9a.2a's private manifest and symbolication are accepted on `development` and Step 9a.2b's product map controls and `package_test` receipt are implemented (acceptance pending); the remaining Step 9 map/custody and release gates remain staged. | | [agent-run-fixture.md](agent-run-fixture.md) | The `relavium agent run --fixture` cassette format — a `StreamChunk[][]` JSON recording of an agent run's LLM stream, for deterministic offline replay. | diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 5473f483..cac20eb8 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -1,6 +1,6 @@ # CLI Packaging Contract -- **Status**: Compiled graph, installed package and Step 8 native activation independently accepted on `development`; the Step 9a.1 candidate build and finalizer are independently accepted on `development` (2026-10-24); Step 9a.2a's strict decoder, private manifest and symbolication are independently accepted on `development` (2026-10-24); the remaining Step 9 map/custody and release gates remain staged +- **Status**: Compiled graph, installed package and Step 8 native activation independently accepted on `development`; the Step 9a.1 candidate build and finalizer are independently accepted on `development` (2026-10-24); Step 9a.2a's strict decoder, private manifest and symbolication are independently accepted on `development` (2026-10-24); Step 9a.2b's product map controls, post-receipt rebuild refusals and `package_test` receipt are implemented, with independent acceptance pending; the remaining Step 9 custody and release gates remain staged - **Date**: 2026-10-15 - **Related**: [ADR-0111](../../decisions/0111-cli-meta-operations-use-a-lazy-packaged-runtime.md), [commands.md](commands.md), [release runbook](../../runbooks/release-a-surface.md) @@ -149,8 +149,8 @@ removal precedes success receipt/summary publication. Worker or cleanup failure prevents publication, and secondary cleanup/diagnostic failure preserves the primary outcome. Private worker identity stays outside public receipts. -Required workflows share one public tarball and a `compiled_package_test` receipt -with current `.nvmrc` Linux/macOS/Windows and exact Node 22.0.0 Linux. Each repeats +Required workflows share one public tarball and a package-test receipt (mode `package_test` +since Step 9a.2b, `compiled_package_test` before it) with current `.nvmrc` Linux/macOS/Windows and exact Node 22.0.0 Linux. Each repeats untouched-entry bootstrap controls and actual workflow, gate, budget, chat/resume, SQLite, native-load, MCP-version and media-cleanup behavior. Their aggregate checks all four actual success records and immutable artifact/source/origin identities. @@ -159,9 +159,9 @@ The [tooling guide](../../../tools/cli-package/README.md) describes invocation a bounded retry evidence. These runners are implemented; required platform execution and independent acceptance are recorded separately in the Wave 3 register. -This intermediate receipt does not claim the full map-bearing package-test mode -below. Original-TypeScript symbolication, private staging and release custody -remain Step 9. Native fatal-owner integration is covered by [Step 8 acceptance](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17). +Since Step 9a.2b the receipt records `package_test`: before writing it the producer verified its +maps and symbolicated actual packed frames (Steps 9a.1 and 9a.2a), and the required `ci` at the +same commit runs the product map controls (below). Release custody remains Steps 9b–9d. Native fatal-owner integration is covered by [Step 8 acceptance](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17). No intermediate local result closes Step 9 or certifies a release. ### Candidate finalization — Step 9 @@ -299,8 +299,8 @@ describes the implemented mechanism; independent acceptance is recorded in the - **Seal.** The seal takes the candidate root, the staging directory and the original checkout's identity explicitly and never resolves a repository from its own location. Public JS must be exactly the staged metafile's runtime outputs, byte-identical to the - finalizer's record; the actual tarball is inspected as in Step 7b. The receipt remains - `compiled_package_test`. + finalizer's record; the actual tarball is inspected as in Step 7b. The receipt remained + `compiled_package_test` until Step 9a.2b flipped it to `package_test`. - **Ordinary map-free control.** The ordinary closure guard refuses any map, map link or unreviewed file in `dist`, and the five build configurations and the CLI configuration are pinned map-free outside a named candidate. @@ -346,7 +346,7 @@ refuses every artifact (the [Interim release guard ruling](../../roadmap/phases/ Step 9a.2a (a [recorded split](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24) of 9a.2, maintainer-approved on 2026-10-24) implements items 4 and 5 above for the package-test -producer. The receipt stays `compiled_package_test` until 9a.2b's product map controls exist. +producer. The receipt stayed `compiled_package_test` until 9a.2b's product map controls existed. "Owner-only" in this section has the meaning the 9a.1 section gives it: mode 0700 or 0600 where POSIX modes are observable; on Windows the private root inherits the temporary directory's ACL, which is not verified. The producer needs Node 22.15 or later, whose `module.registerHooks` the @@ -412,6 +412,65 @@ import` makes no network request, but nothing enforces that. This is a named lim package-test producer, owned by [deferred-tasks.md](../../roadmap/deferred-tasks.md#wave-3-step-9a2a--recorded-low-findings-2026-10-24). +### Product map controls and the package-test receipt — Step 9a.2b + +Step 9a.2b (the [preflight decisions](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-preflight-decisions--2026-10-24), +maintainer-approved on 2026-10-24) closes ADR-0111's post-receipt rebuild refusal for the +package-test producer, adds the product map controls and only then flips the receipt. + +- **Rebuild refusals.** A second prepack into a sealed staging refuses before any build + (`STAGING_NOT_EMPTY`). `sealPackage` refuses with `PACKED_CLI_ALREADY_SEALED` at its entry, + before the inspection and any capture, when the artifact directory holds anything besides the one + tarball or the staging already holds a manifest (checked without following a link); a receipt + another seal wrote meanwhile refuses with the same code. Immediately before the receipt is written, + `verifySealedDist` compares the whole `dist` inventory, every output's digest and the tarball's + digest with the manifest just sealed (`PACKED_CLI_MANIFEST_DIST_DRIFT`, + `PACKED_CLI_MANIFEST_TARBALL_CHANGED`), so a `dist` rebuilt or changed after the inspection refuses, + also in a chunk no frame lands in. Release-mode retry and no-rebuild rules stay with Steps 9c and 9d. +- **Product map controls.** `pnpm check:map-controls` (`tools/cli-package/map-controls.mjs`) runs one + real package-test production at the checked-out commit through the producer's own ports. Inside its + seal, after the baseline receipt and while the private staging still exists, each control corrupts + one product copy — the owned candidate tree, or a copy of its private staging re-recorded so that + only the layer under test can refuse — requires that control's exact refusal and restores the file + by digest: + - a second prepack into the sealed staging, and a re-seal of the artifact directory or of the + sealed staging beside a copied tarball; + - on every anchor-bearing chunk, wrong JS by line and by column, a map from a build one line apart + and rotated map sources (`PACKED_CLI_SYMBOLICATION_ANCHOR_UNMAPPED`), after a re-recorded + unchanged copy is accepted; + - a wrong source revision, clean (`PACKED_CLI_SYMBOLICATION_SOURCE_REVISION`) and dirty + (`ANCHOR_UNMAPPED`); + - a changed chunk no frame lands in, and a real rebuild from a changed source, against the sealed + manifest (`PACKED_CLI_MANIFEST_DIST_DRIFT`), after the sealed `dist` verifies; + - an upstream map that is missing, malformed (an extra `sourcesContent`, an out-of-range source + index) or cut inside a VLQ value, each refused by the finalizer with its own fixed line after the + bundler silently succeeded (one build, nothing finalized, staged or packed); a byte-truncated + upstream map, which fails the bundler's own build; and a shifted but strictly valid upstream map, + which the finalizer accepts and symbolication refuses. + + A restoration production must then seal again; its tarball digest is not compared with the + baseline's (see deferred-tasks.md, "Two packs of one source"). Every copy stays under the private + staging, which the producer removes first on success, failure and interrupt, and the developer's + checkout must be unchanged before and after. The runner prints one JSON summary of control ids and + expected and observed codes or fixed prepack lines — never a path, frame, position, map byte or + child output — and exits 1 on any mismatch. It runs in `pnpm run ci` right after + `smoke:cli-package`, and as the required `map-controls` CI job: ubuntu, at most 20 minutes, read-only + permissions, no remote cache credentials, no pnpm cache, no build step and no upload, inside `ci`'s + needs with a guard that refuses every result except success (the required contexts stay `ci` and + `coverage`). That job's production is a second, independent build that is never shipped, not a + retry of the packed candidate, so ADR-0111's no-rebuild rule for a candidate is not touched. +- **The `package_test` receipt.** The seal and both receipt validators share one constant, the earlier + `compiled_package_test` is refused, and the receipt keeps version 1 and its twelve fields; + consumers and the aggregate verify identity through the shared validator. The mode is a claim, not + evidence of the map checks: what backs it is the required `ci` run at the same commit, and Steps + 9c and 9d must not read it as proof. `release.yml` runs no map controls of its own (its graph is + Step 9d's); a manual dry-run's `package_test` artifact is backed by the required `ci` run at the same + commit, and the interim guard refuses every release-workflow artifact anyway. + +The runner keeps private maps and manifests longer than the producer does, always under the staging; a +runner killed outright can leave the owner-only root behind, the producer's existing named limit. The +CI job runs on Linux only; a local run on Windows claims no mode evidence, as above. + ## Recipient custody The maintainer selected encrypted CI archival. GnuPG is release tooling, never a @@ -439,9 +498,10 @@ This wave creates no automatic deletion policy or production key. **Package-test mode** is required on PR/push and manual dry-run. In its isolated uncached originating pack job, build once, inspect exact tarball, verify maps and CLI/engine symbolication locally, then discard all private maps/manifests/metafiles. -Share only the tarball and a nonsecret package-test receipt (version, mode -package_test, source SHA, package version, tarball digest and originating workflow -identity) with platform consumers. It needs no production recipient. GPG fixture +Share only the tarball and a nonsecret package-test receipt (version 1 with mode +`package_test`; the source SHA and dirty flag; the package version; the tarball's name, +digest and size; the public file inventory and package metadata; the originating workflow +identity and the producer toolchain) with platform consumers. It needs no production recipient. GPG fixture controls remain separate and use synthetic map payloads only. A package-test artifact is never promoted to a release candidate after private diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index 3e765762..24fac7d8 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -138,7 +138,7 @@ The [deferred register](deferred-tasks.md) retains its existing limits; closure **Wave 3 Steps 1–8 merged to `main` on 2026-10-18 as [PR #95](https://github.com/HodeTech/Relavium/pull/95), merge commit `6d2a7372` (22:44 UTC). Wave 3 itself is not complete: Steps 9–19 had not started at -that merge (Steps 9a.1 and 9a.2a are now accepted on `development`; Steps 9a.2b–19 have not started) and whole-wave acceptance +that merge (Steps 9a.1 and 9a.2a are now accepted on `development` and Step 9a.2b is implemented and under review; Steps 9b–19 have not started) and whole-wave acceptance remains Step 19.** The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) joins the completed scope, every review attribution, the independent review's dispositions and the publication state observed at merge. It was the maintainer-authorized, scoped publication checkpoint. @@ -172,7 +172,7 @@ bar and round cap, as its medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, by the author alone). Its other findings are recorded in -[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is independently accepted on `development` at `cf97b7f1` (2026-10-24): its round 13 reported no blocker, high or medium finding and its security sitting was clean, and the round's low findings are recorded in deferred-tasks.md ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)); Step 9a.2 is split into 9a.2a and 9a.2b, and Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24) after three rounds, the third with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)); Step 9a.2b is next; the substrate's +[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is independently accepted on `development` at `cf97b7f1` (2026-10-24): its round 13 reported no blocker, high or medium finding and its security sitting was clean, and the round's low findings are recorded in deferred-tasks.md ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)); Step 9a.2 is split into 9a.2a and 9a.2b, and Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24) after three rounds, the third with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)); Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is implemented on `development` and under review ([checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-implementation-checkpoint--2026-10-24)); the substrate's 9a.1–9d increments carry a dedicated archive/crypto security sitting. Steps 10–19 follow the [increment table](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) and end with whole-wave acceptance at Step 19. `development` is pushed after each accepted increment; PR #95 no longer bounds the @@ -1141,7 +1141,7 @@ an ESLint selector rather than a list. ### Wave 3 — Clear the ground -**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Steps 9a.1 and 9a.2a accepted on `development` (2026-10-24), Steps 9a.2b–19 NOT STARTED — 2026-10-24.** +**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Steps 9a.1 and 9a.2a accepted on `development` (2026-10-24), Step 9a.2b under review, Steps 9b–19 NOT STARTED — 2026-10-24.** ADR-0110–0113, their linked contracts and the nineteen-increment plan were approved on 2026-10-16. [Steps 1–8 were independently accepted](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) on `development`, with whole Step 8 reviewed at `c83c8575`, and published as the scoped checkpoint diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index cf4559d7..5e6958ca 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -2,7 +2,7 @@ > Status: Living -> Last updated: 2026-10-24 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Step 9a.1 is independently accepted on `development` with its recorded low findings below, Step 9a.2a is independently accepted on `development` with its recorded low findings below, Steps 9a.2b–19 not started; the PR95 increment is closed on `development` under the maintainer's 2026-10-21 round cap). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's edge cases with the maintainer's 2026-10-21 acceptance bar and round cap and the 2026-10-22 rulings. Existing approved deferrals remain. +> Last updated: 2026-10-24 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Step 9a.1 is independently accepted on `development` with its recorded low findings below, Step 9a.2a is independently accepted on `development` with its recorded low findings below, Step 9a.2b is implemented and under review, Steps 9b–19 not started; the PR95 increment is closed on `development` under the maintainer's 2026-10-21 round cap). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's edge cases with the maintainer's 2026-10-21 acceptance bar and round cap and the 2026-10-22 rulings. Existing approved deferrals remain. > The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) records completed prerequisites; full M/N lifecycle and other scheduled work remain with their owners. > > Earlier triage: the Phase-2.6 rewrite triaged every open item, the 2026-07-19 full-project diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index a7dcc8e6..9af2e4cf 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -3700,7 +3700,7 @@ No credential values or request captures belong in tracked evidence. | 9a.1 | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — candidate build and finalizer:** an owned isolated candidate copy, fresh uncached upstream maps, candidate-mode tsup selection with absent `sourcesContent`, the prepack finalizer (upstream and final map validation, repository-relative sources, exact trailer removal before hashes), owner-only private staging outside public artifacts and caches, closure/seal adaptation, the ordinary map-free control, the PR #95 archive/worker fold-ins, and a fail-closed `release.yml` publish guard that refuses any non-release candidate until 9d and Wave 4b custody exist. The receipt stays `compiled_package_test`. *Implemented on `development` on 2026-10-22 ([checkpoint](#wave-3-step-9a1-implementation-checkpoint--2026-10-22)); independently accepted at `cf97b7f1` on 2026-10-24 ([acceptance](#wave-3-step-9a1-acceptance--2026-10-24)).* | | 9a.2 | **Recorded split — manifest, symbolication and package-test receipt:** the private map manifest keyed by `dist/.js`, actual packed CLI and engine frame symbolication against the matching source revision, wrong JS/map/source and missing/malformed-upstream negative controls (product copies via `check:map-controls`, also a required uncached CI job), and the atomic `package_test` receipt flip across seal, binding, consumers, aggregate and both workflows. *Split on 2026-10-24 (maintainer-approved; ID coverage #46/ADR-0111 unchanged) into 9a.2a and 9a.2b below, which replace it as increments ([preflight decisions](#wave-3-step-9a2-preflight-decisions--2026-10-24)).* | | 9a.2a | **Recorded split, 2026-10-24 (maintainer-approved) — decoder, private manifest and symbolication:** an in-house strict source-map decoder; strict mapping and range validation of upstream and final maps in the finalizer; the private map manifest (`staging/manifest.json`, keyed by `dist/.js`, with a per-source digest inventory and the commit's tree), sealed after the archive inspection; real packed CLI and `@relavium/core` frame capture through a tooling-only preload during an offline `relavium import`, run on the finalized `dist` that the inspection proves byte-equal to the tarball; symbolication of those frames against the matching source revision (clean: the checkout's Git objects; dirty, local only: the candidate copy), wired into the seal before the receipt is written. The receipt stays `compiled_package_test`. Security sitting. *Independently accepted on `development` at `a92a7602` on 2026-10-24 ([checkpoint](#wave-3-step-9a2a-implementation-checkpoint--2026-10-24), [acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)).* | -| 9a.2b | **Recorded split, 2026-10-24 (maintainer-approved) — product map controls and the package-test receipt:** `check:map-controls` over product copies (wrong JS by line and column, wrong map, permuted sources, wrong source, missing, malformed and truncated upstream maps, a shifted valid upstream map, and the package-test producer's post-receipt rebuild refusals that ADR-0111 lists), run in `pnpm run ci` and as a required uncached CI job inside `ci`'s needs (the required contexts stay `ci` and `coverage`); then the atomic `package_test` flip across seal, binding, consumers, aggregate and both workflows, version 1 kept. Security sitting. *Preflight decisions approved on 2026-10-24 ([decisions](#wave-3-step-9a2b-preflight-decisions--2026-10-24)).* | +| 9a.2b | **Recorded split, 2026-10-24 (maintainer-approved) — product map controls and the package-test receipt:** `check:map-controls` over product copies (wrong JS by line and column, wrong map, permuted sources, wrong source, missing, malformed and truncated upstream maps, a shifted valid upstream map, and the package-test producer's post-receipt rebuild refusals that ADR-0111 lists), run in `pnpm run ci` and as a required uncached CI job inside `ci`'s needs (the required contexts stay `ci` and `coverage`); then the atomic `package_test` flip across seal, binding, consumers, aggregate and both workflows, version 1 kept. Security sitting. *Preflight decisions approved on 2026-10-24 ([decisions](#wave-3-step-9a2b-preflight-decisions--2026-10-24)); implemented on `development` on 2026-10-24, acceptance pending ([checkpoint](#wave-3-step-9a2b-implementation-checkpoint--2026-10-24)).* | | 9b | **Recorded split — recipient custody:** synthetic-only isolated GnuPG controls (public-only home/keyring, pinned primary and encryption-subkey fingerprints, machine-readable refusal of missing, expired, revoked or unusable recipients), encrypt once and reuse the original ciphertext. Explicit crypto/custody security sitting. Fixture proof never claims production recipient or custody configuration. | | 9c | **Recorded split — offline release state controller:** permission-enforcing offline GitHub/npm controller, permissioned prebuild discovery and reservation, and every rerun, partial, conflict and expiry state, with no overwrite, rebuild or re-encryption on uncertain history. | | 9d | **Recorded split — workflow, runbook and whole-Step-9 landing:** least-privilege actual workflow dependency graph, required artifact/platform aggregation and exact-artifact consumers, lost-staging refusal and the release runbook update. The whole-Step-9 cumulative review retains all 9a–9d changes with preservation traces into Step 8, and the dedicated archive/crypto security sitting closes the step. | @@ -9796,3 +9796,24 @@ Engineering defaults recorded with the rulings: each control asserts its exact r anchor-bearing chunk; permuted sources rotate with no fixed point; the shifted upstream map is checked to be strictly valid before use; the runner prints one JSON summary with codes and outcomes only — no path, frame, map byte or child output — and compares bytes by digest; it runs in `pnpm run ci` right after `smoke:cli-package`. + +### Wave 3 Step 9a.2b implementation checkpoint — 2026-10-24 + +**Scope and source.** Step 9a.2b (the 9a.2b row; #46, ADR-0111) implements, on `development` after the 9a.2a +acceptance and the 9a.2b preflight decisions (`436d7c67`), four ordered commits and this record: the seal-side +rebuild refusals (`PACKED_CLI_ALREADY_SEALED` at `sealPackage`'s entry and `verifySealedDist` before the +receipt, `seal.mjs` and `manifest.mjs`, with a prepack test of a second prepack into a sealed staging); the +product map controls runner (`tools/cli-package/map-controls.mjs`, `pnpm check:map-controls`, with tests of its +pure helpers); the required `map-controls` CI job inside `ci`'s needs with its success guard, `pnpm run ci`'s +step after `smoke:cli-package` and the workflow pins; and only then the `package_test` flip (one +`PACKAGE_TEST_MODE` shared by the seal and both receipt validators, `compiled_package_test` refused), with the +[packaging contract](../../reference/cli/packaging.md#product-map-controls-and-the-package-test-receipt--step-9a2b), +the tooling README and the status lines. No runtime dependency or ADR is added; `release.yml` gets no map +controls job, as ruled. On this host the runner's 24 controls pass in about 30 seconds on one real production, +`smoke:cli-package` passes with the flipped receipt through the real producer and consumers, and +`test:acceptance-tools` passes 911 of 911. + +**Remaining gates.** The internal rigor pass (a whole-increment mutation sweep in which each control must fail +when its refusal is removed, the claims, platform and security lenses), the full `pnpm run ci`, then one NEW +complete cumulative round with no blocker, high or medium finding and no regression, together with the separate +scoped security sitting at the same head and bar, and the `map-controls` job green on the push CI. diff --git a/tools/cli-package/README.md b/tools/cli-package/README.md index 49157f4f..7ebb690a 100644 --- a/tools/cli-package/README.md +++ b/tools/cli-package/README.md @@ -39,7 +39,8 @@ An exact public allowlist, checksums, modes, hashbangs, byte ceilings and negati controls exclude links, traversal, duplicates, extra files, maps and private build inventories. This deliberately supports the pinned packer's dialect, not general tar. -The resulting public `package-test-receipt.json` records `compiled_package_test`. +The resulting public `package-test-receipt.json` records `package_test` (Step 9a.2b; it recorded +`compiled_package_test` before the product map controls existed). The candidate's maps are built, validated and then removed with the rest of the private staging, also when the producer is interrupted (SIGINT, SIGTERM, SIGHUP), which stops the build first and empties the artifact directory, even after the last build child @@ -68,7 +69,12 @@ native addons in its own process, and needs Node 22.15 or later; an older produc the matching source revision (`symbolicate.mjs`). The manifest is discarded with the staging, and the capture's home, project and frames with the private root. The receipt does **not** certify encrypted custody or a production release candidate (Steps 9b–9d), -and it stays `compiled_package_test` until the product map controls of Step 9a.2b. No private map, +and its `package_test` mode is a claim, backed by the required `ci` run at the same commit. A sealed +candidate is never sealed again (`PACKED_CLI_ALREADY_SEALED`), and `dist` and the tarball must still be +the manifest's when the receipt is written (`verifySealedDist`). `pnpm check:map-controls` +(`map-controls.mjs`) runs every named negative control on product copies of one real production and +prints one JSON summary of codes, never a path or map byte; the required `map-controls` CI job runs it +uncached and uploads nothing. No private map, metafile, manifest or per-source inventory enters the public artifact. The release workflow's `release-guard.mjs` refuses every artifact until verified release custody exists: it sets its exit code to 1 before it evaluates its rule (`release-guard-core.mjs`), so a rule From 9f8b08900eb3b19ec245d94f1f5c7542e231494b Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sat, 24 Oct 2026 20:33:24 +0300 Subject: [PATCH 49/97] docs(repo): record 9a.2a's push CI and the native-identity increment Step 9a.2a's push CI (run 40105286543) passed all sixteen required jobs. The advisory Windows job failed on a security assertion unrelated to 9a.2a: the filesystem policy read a replaced, granted subdirectory instead of refusing it, the test's first failure in sixty runs. The cause is in the code: directory and file identities are compared by dev and ino read without { bigint: true }, and on Windows ino is a 64-bit NTFS file ID that a JavaScript number cannot hold exactly above 2^53, so a replacement next to the original can round to the same identity. Five identity checks share the pattern (the filesystem policy's pins, the workflow catalog, and the media store's pins, read check and temporary claims). The maintainer approved a separate increment for it on 2026-10-24, after 9a.2b and before 9b. The register gains row NI and the 9a.2a acceptance its push-CI paragraph, and deferred-tasks.md tracks the finding until the increment lands. Co-Authored-By: Claude Opus 5.5 --- docs/roadmap/deferred-tasks.md | 19 ++++++++++++++++++- .../phase-2.5.5-hardening-and-remediation.md | 14 ++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index 5e6958ca..8ee099c3 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -719,7 +719,24 @@ names both `produce.mjs` changes, INT-9A2A-R3-1); the two test gaps are recorded **Re-check:** add an anchor whose source the manifest maps and whose call site resolves in the source revision but that no captured frame reaches, and expect `ANCHOR_UNMAPPED`. *(nit · tools/cli-package/symbolicate.test.mjs)* -## Phase 2.6.5 deferrals — `CR-93` and `CR-95` (long half), recorded 2026-09-06; `CR-70`, `CR-94` (remainder) and the in-turn overflow continuation, recorded 2026-09-14; the errored-or-aborted turn residual, the pre-upgrade free-page bytes and the estimator's calibration, recorded 2026-09-18 +## Native file identity on Windows — approved increment (2026-10-24) + +Found by Step 9a.2a's push CI (the advisory Windows job of run 40105286543). The maintainer approved a separate +increment, after 9a.2b and before 9b (register row NI), rather than a deferral; this entry tracks it until then. + +- [ ] **Identity checks compare `ino` as a JavaScript number, which Windows file IDs overflow.** *High; security, + Windows.* `statSync`, `lstatSync`, `fstatSync` and `FileHandle.stat` without `{ bigint: true }` return `ino` as a + number, but on Windows `ino` is the volume's 64-bit file ID; above 2^53 neighbouring IDs round to the same value, + so a replaced file or directory can compare as the original. `apps/cli/src/engine/tool-host/fs-policy.test.ts`, + "revokes a selected subdirectory replacement even when its enclosing grant is unchanged", read the replacement on + the Windows runner (its first failure in sixty runs). Sites: `apps/cli/src/engine/tool-host/fs-policy.ts` + (`pinDirectory`/`validatePin`), `apps/cli/src/workflows/catalog.ts` (the open check), and + `packages/db/src/media-store-paths.ts` (directory pins and `assertOwnedFile`), `media-store-read.ts` (the + read's before/after check) and `media-store-temporaries.ts` (`same` and the `Number(...)` comparison in + `nativeFile`). **Re-check:** every identity is read with `{ bigint: true }` and compared as `bigint` (or its exact + string), with a replacement test per site that fails when the comparison goes back to a number; the Windows job + passes the fs-policy test repeatedly. *(high · the five files above)* +, recorded 2026-09-06; `CR-70`, `CR-94` (remainder) and the in-turn overflow continuation, recorded 2026-09-14; the errored-or-aborted turn residual, the pre-upgrade free-page bytes and the estimator's calibration, recorded 2026-09-18 **Phase closure — 2026-10-11:** W8 merged to `main` in PR #92 and Phase 2.6.5 is complete with approved deferrals, 49/51 items closed. The diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 9af2e4cf..4ce1cf14 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -3701,6 +3701,7 @@ No credential values or request captures belong in tracked evidence. | 9a.2 | **Recorded split — manifest, symbolication and package-test receipt:** the private map manifest keyed by `dist/.js`, actual packed CLI and engine frame symbolication against the matching source revision, wrong JS/map/source and missing/malformed-upstream negative controls (product copies via `check:map-controls`, also a required uncached CI job), and the atomic `package_test` receipt flip across seal, binding, consumers, aggregate and both workflows. *Split on 2026-10-24 (maintainer-approved; ID coverage #46/ADR-0111 unchanged) into 9a.2a and 9a.2b below, which replace it as increments ([preflight decisions](#wave-3-step-9a2-preflight-decisions--2026-10-24)).* | | 9a.2a | **Recorded split, 2026-10-24 (maintainer-approved) — decoder, private manifest and symbolication:** an in-house strict source-map decoder; strict mapping and range validation of upstream and final maps in the finalizer; the private map manifest (`staging/manifest.json`, keyed by `dist/.js`, with a per-source digest inventory and the commit's tree), sealed after the archive inspection; real packed CLI and `@relavium/core` frame capture through a tooling-only preload during an offline `relavium import`, run on the finalized `dist` that the inspection proves byte-equal to the tarball; symbolication of those frames against the matching source revision (clean: the checkout's Git objects; dirty, local only: the candidate copy), wired into the seal before the receipt is written. The receipt stays `compiled_package_test`. Security sitting. *Independently accepted on `development` at `a92a7602` on 2026-10-24 ([checkpoint](#wave-3-step-9a2a-implementation-checkpoint--2026-10-24), [acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)).* | | 9a.2b | **Recorded split, 2026-10-24 (maintainer-approved) — product map controls and the package-test receipt:** `check:map-controls` over product copies (wrong JS by line and column, wrong map, permuted sources, wrong source, missing, malformed and truncated upstream maps, a shifted valid upstream map, and the package-test producer's post-receipt rebuild refusals that ADR-0111 lists), run in `pnpm run ci` and as a required uncached CI job inside `ci`'s needs (the required contexts stay `ci` and `coverage`); then the atomic `package_test` flip across seal, binding, consumers, aggregate and both workflows, version 1 kept. Security sitting. *Preflight decisions approved on 2026-10-24 ([decisions](#wave-3-step-9a2b-preflight-decisions--2026-10-24)); implemented on `development` on 2026-10-24, acceptance pending ([checkpoint](#wave-3-step-9a2b-implementation-checkpoint--2026-10-24)).* | +| NI | **Native file identity on Windows (maintainer-approved 2026-10-24; after 9a.2b, before 9b):** every file and directory identity check compares `dev`/`ino` read with `{ bigint: true }` — the filesystem policy's pinned roots, the workflow catalog's open check and the media store's pins, read check and temporary claims — so a replaced file or directory can never compare equal on Windows, where `ino` is a 64-bit NTFS file ID that a JavaScript number cannot hold exactly; a replacement test per site. Found by 9a.2a's push CI ([acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)); recorded in [deferred-tasks.md](../deferred-tasks.md#native-file-identity-on-windows--approved-increment-2026-10-24). Security sitting. | | 9b | **Recorded split — recipient custody:** synthetic-only isolated GnuPG controls (public-only home/keyring, pinned primary and encryption-subkey fingerprints, machine-readable refusal of missing, expired, revoked or unusable recipients), encrypt once and reuse the original ciphertext. Explicit crypto/custody security sitting. Fixture proof never claims production recipient or custody configuration. | | 9c | **Recorded split — offline release state controller:** permission-enforcing offline GitHub/npm controller, permissioned prebuild discovery and reservation, and every rerun, partial, conflict and expiry state, with no overwrite, rebuild or re-encryption on uncertain history. | | 9d | **Recorded split — workflow, runbook and whole-Step-9 landing:** least-privilege actual workflow dependency graph, required artifact/platform aggregation and exact-artifact consumers, lost-staging refusal and the release runbook update. The whole-Step-9 cumulative review retains all 9a–9d changes with preservation traces into Step 8, and the dedicated archive/crypto security sitting closes the step. | @@ -9772,6 +9773,19 @@ second) matched this checkpoint. The round's three verified findings (one low, t corrected in the acceptance commit, which changes no code, and the two test gaps stay open there. The receipt stays `compiled_package_test`; Step 9a.2b is next. +**Push CI.** `development` was pushed at `f65999a6` (the acceptance commit on top of `a92a7602`). All sixteen +required jobs of [run 40105286543](https://github.com/HodeTech/Relavium/actions/runs/40105286543) passed, among them +the originating pack, the four same-tarball installed-CLI consumers and their aggregate, lint, typecheck and test, +the engine coverage floor and the causal-control families. The advisory Windows job failed on a security assertion +unrelated to 9a.2a: `apps/cli/src/engine/tool-host/fs-policy.test.ts`, "revokes a selected subdirectory +replacement even when its enclosing grant is unchanged", read the replacement instead of refusing it. It was the +first failure of that test in the last sixty CI runs. The cause is in the code: the filesystem policy pins a +directory by `dev` and `ino` read without `{ bigint: true }`, and on Windows `ino` is a 64-bit NTFS file ID that a +JavaScript number above 2^53 cannot hold exactly, so a directory created next to the renamed original can round to +the same identity. Four other identity checks share the pattern. The maintainer approved a separate increment for +them on 2026-10-24, after 9a.2b and before 9b (row NI; +[deferred-tasks.md](../deferred-tasks.md#native-file-identity-on-windows--approved-increment-2026-10-24)). + ### Wave 3 Step 9a.2b preflight decisions — 2026-10-24 A read-only preflight (five readers and a synthesis at `f65999a6`; the detailed record and a Turkish decision From 3e6ae33c8d643f484c16e3248406eebae12f9c94 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sat, 24 Oct 2026 20:56:22 +0300 Subject: [PATCH 50/97] fix(repo): close the Step 9a.2b internal rigor pass The rigor pass over 77ee5d0a (three mutation sweeps, claims, platform, security and integration lenses and a critic) confirmed one high and several medium findings; all are corrected. - High: the re-seal-receipt control re-sealed against the sealed staging, so the staging half of ALREADY_SEALED refused for it and the artifact half could be removed unnoticed. It now uses a copy of the staging without its manifest; each re-seal arm isolates one half. - The seal's default verify port is now tested (the real check refuses a changed chunk or tarball), and a new seal-verify control changes a chunk during the seal's own run. - The runner requires the complete expected control set (seventeen plus four per anchor-bearing chunk, each once) through a tested verdict and report; map-controls.mjs is now an unconditional entry point over map-controls-core.mjs, so it cannot exit 0 without running, as its path-comparing guard could on Windows. - Each per-chunk control must change exactly its own chunk, in the way it names; a half-configured CI environment refuses before anything is built; a refused control build's dist is removed at once. - Workflow pins: no custom or default shells, the exact job environment and step keys, and negative cases for the guard's result, a tolerated step or guard, permissions, platform, needs, a matrix and pnpm inputs. - Tests for the already-sealed refusal's order, a failed receipt write, the version's start anchor and verifySealedDist's outputs and bounds. - Docs: the already-sealed condition, the summary and checkout wording, and the dry-run claim narrowed to when a ci run at exactly that commit backs an artifact; five lows recorded in deferred-tasks.md, two for 9d. Gates: lint:tools, typecheck:tools, prettier, test:acceptance-tools (932 of 932 under CI=true GITHUB_ACTIONS=true), the real runner (25 of 25 expected controls, 31 s) and smoke:cli-package. Refs: ADR-0111, #46 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 4 +- .github/workflows/release.yml | 5 +- docs/reference/cli/README.md | 2 +- docs/reference/cli/packaging.md | 97 +- docs/roadmap/deferred-tasks.md | 36 + .../phase-2.5.5-hardening-and-remediation.md | 32 +- tools/ci/require-packed-cli.test.mjs | 54 +- tools/ci/require-w8-acceptance.test.mjs | 1 + tools/cli-package/README.md | 15 +- tools/cli-package/binding.test.mjs | 8 + tools/cli-package/manifest.test.mjs | 54 ++ tools/cli-package/map-controls-core.mjs | 826 ++++++++++++++++++ tools/cli-package/map-controls.mjs | 704 +-------------- tools/cli-package/map-controls.test.mjs | 75 +- tools/cli-package/seal.mjs | 5 +- tools/cli-package/seal.test.mjs | 114 +++ 16 files changed, 1284 insertions(+), 748 deletions(-) create mode 100644 tools/cli-package/map-controls-core.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ae00c15..e5fef3a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -644,8 +644,8 @@ jobs: # The runner builds inlined dependencies fresh in an owned candidate tree, then prepack performs the sole # final CLI build, where original-TS maps are built, validated, staged privately and discarded (Wave 3 # Step 9a.1); the seal then writes the private manifest and symbolicates actual packed frames, both - # discarded too (Step 9a.2a); only the tarball and its package_test receipt leave this job. The map-controls - # job below supplies the negative controls behind that mode (Step 9a.2b). + # discarded too (Step 9a.2a); only the tarball and its package_test receipt leave this job. The mode is a + # claim: the map-controls job below runs the negative controls, and `ci` requires its success (Step 9a.2b). - name: Pack once and independently inspect the public package env: RELAVIUM_PACKAGE_SOURCE_SHA: ${{ github.sha }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9edcb539..99de02f8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -102,8 +102,9 @@ jobs: # The same required package runner owns fresh dependencies and the sole prepack CLI build in an owned # candidate tree, where original-TS maps are built, validated, staged privately and discarded (Wave 3 # Step 9a.1), with the private manifest and the packed-frame symbolication (Step 9a.2a); their - # encrypted release custody is Steps 9b–9d. The receipt records package_test; the product map controls - # behind that mode run in the required CI workflow at the same commit (Step 9a.2b), not in this one. + # encrypted release custody is Steps 9b–9d. The receipt records package_test, a claim: the product map + # controls run only in the CI workflow, and back an artifact only where a successful CI run exists at + # exactly its commit (Step 9a.2b; packaging.md). - name: Pack once and independently inspect the public package env: RELAVIUM_PACKAGE_SOURCE_SHA: ${{ github.sha }} diff --git a/docs/reference/cli/README.md b/docs/reference/cli/README.md index 8c1ea913..3867ce53 100644 --- a/docs/reference/cli/README.md +++ b/docs/reference/cli/README.md @@ -13,5 +13,5 @@ Part of [reference/](../README.md). | [commands.md](commands.md) | The full `relavium` command surface (run, chat, chat-resume/list/export, agent run, list, create, import, export, logs, gate, init, provider) plus interactive TUI vs CI JSON mode. | | [home.md](home.md) | The bare-invocation interactive **Home** — the TTY gate, the read-only management strip over `history.db`, the Home↔chat mode machine + keymap, bracketed paste, the min-terminal degrade, and the signal/exit-code lifecycle (2.5.B / ADR-0054). | | [chat-session.md](chat-session.md) | The `relavium chat` agent-session REPL — entry, agent/model selection, the multi-turn loop, streaming, tool availability, the `--json` session-event stream, and exit code 4. | -| [packaging.md](packaging.md) | Wave 3 finite runtime graph, installed-package consumers and Step 8 native activation accepted on `development`; the Step 9a.1 candidate build and finalizer are accepted on `development` and Step 9a.2a's private manifest and symbolication are accepted on `development` and Step 9a.2b's product map controls and `package_test` receipt are implemented (acceptance pending); the remaining Step 9 map/custody and release gates remain staged. | +| [packaging.md](packaging.md) | Wave 3 finite runtime graph, installed-package consumers and Step 8 native activation accepted on `development`; the Step 9a.1 candidate build and finalizer are accepted on `development` and Step 9a.2a's private manifest and symbolication are accepted on `development` and Step 9a.2b's product map controls and `package_test` receipt are implemented (acceptance pending); the remaining Step 9 custody and release gates remain staged. | | [agent-run-fixture.md](agent-run-fixture.md) | The `relavium agent run --fixture` cassette format — a `StreamChunk[][]` JSON recording of an agent run's LLM stream, for deterministic offline replay. | diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index cac20eb8..d2e782d4 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -419,29 +419,35 @@ maintainer-approved on 2026-10-24) closes ADR-0111's post-receipt rebuild refusa package-test producer, adds the product map controls and only then flips the receipt. - **Rebuild refusals.** A second prepack into a sealed staging refuses before any build - (`STAGING_NOT_EMPTY`). `sealPackage` refuses with `PACKED_CLI_ALREADY_SEALED` at its entry, - before the inspection and any capture, when the artifact directory holds anything besides the one - tarball or the staging already holds a manifest (checked without following a link); a receipt - another seal wrote meanwhile refuses with the same code. Immediately before the receipt is written, - `verifySealedDist` compares the whole `dist` inventory, every output's digest and the tarball's - digest with the manifest just sealed (`PACKED_CLI_MANIFEST_DIST_DRIFT`, + (`STAGING_NOT_EMPTY`). `sealPackage` refuses with `PACKED_CLI_ALREADY_SEALED` at its entry, before + the inspection and any capture, when a receipt already lies beside the tarball or the staging already + holds a manifest (checked without following a link); any other extra file in the artifact directory + keeps the public artifact check's own refusal (`PACKED_CLI_UNEXPECTED_PUBLIC_ARTIFACT`), and a receipt + another seal wrote meanwhile refuses with the same code. The refusal is keyed on the artifact + directory and the staging, not on the candidate's identity. Immediately before the receipt is + written, `verifySealedDist` compares the whole `dist` inventory, every output's digest and the + tarball's digest with the manifest just sealed (`PACKED_CLI_MANIFEST_DIST_DRIFT`, `PACKED_CLI_MANIFEST_TARBALL_CHANGED`), so a `dist` rebuilt or changed after the inspection refuses, - also in a chunk no frame lands in. Release-mode retry and no-rebuild rules stay with Steps 9c and 9d. -- **Product map controls.** `pnpm check:map-controls` (`tools/cli-package/map-controls.mjs`) runs one - real package-test production at the checked-out commit through the producer's own ports. Inside its - seal, after the baseline receipt and while the private staging still exists, each control corrupts - one product copy — the owned candidate tree, or a copy of its private staging re-recorded so that - only the layer under test can refuse — requires that control's exact refusal and restores the file - by digest: - - a second prepack into the sealed staging, and a re-seal of the artifact directory or of the - sealed staging beside a copied tarball; + also in a chunk whose map names no anchor source. Release-mode retry and no-rebuild rules stay with + Steps 9c and 9d. +- **Product map controls.** `pnpm check:map-controls` (`tools/cli-package/map-controls.mjs`, an + unconditional entry point over `map-controls-core.mjs`) runs one real package-test production at the + checked-out commit through the producer's own ports. A half-configured CI environment refuses before + anything is built (`PACKED_CLI_MAP_CONTROL_ORIGIN`). Inside the seal, after the baseline receipt and + while the private staging still exists, each control corrupts one product copy — the owned candidate + tree, or a copy of its private staging re-recorded so that only the layer under test can refuse — + requires that control's exact refusal and restores the file by digest: + - a second prepack into the sealed staging; a re-seal of the artifact directory against a copy of + the staging without its manifest, and of the sealed staging beside a tarball copied without its + receipt, each isolating one half of `PACKED_CLI_ALREADY_SEALED` with no capture run; - on every anchor-bearing chunk, wrong JS by line and by column, a map from a build one line apart - and rotated map sources (`PACKED_CLI_SYMBOLICATION_ANCHOR_UNMAPPED`), after a re-recorded - unchanged copy is accepted; + and rotated map sources (`PACKED_CLI_SYMBOLICATION_ANCHOR_UNMAPPED`), each checked to have changed + exactly its own chunk in the way it names, after a re-recorded unchanged copy is accepted; - a wrong source revision, clean (`PACKED_CLI_SYMBOLICATION_SOURCE_REVISION`) and dirty (`ANCHOR_UNMAPPED`); - - a changed chunk no frame lands in, and a real rebuild from a changed source, against the sealed - manifest (`PACKED_CLI_MANIFEST_DIST_DRIFT`), after the sealed `dist` verifies; + - a changed chunk whose map names no anchor source, and a real rebuild from a changed source, against + the sealed manifest (`PACKED_CLI_MANIFEST_DIST_DRIFT`), after the sealed `dist` verifies; and the + seal's own check, which refuses a chunk the run changes after the inspection, writing no receipt; - an upstream map that is missing, malformed (an extra `sourcesContent`, an out-of-range source index) or cut inside a VLQ value, each refused by the finalizer with its own fixed line after the bundler silently succeeded (one build, nothing finalized, staged or packed); a byte-truncated @@ -449,27 +455,38 @@ package-test producer, adds the product map controls and only then flips the rec which the finalizer accepts and symbolication refuses. A restoration production must then seal again; its tarball digest is not compared with the - baseline's (see deferred-tasks.md, "Two packs of one source"). Every copy stays under the private - staging, which the producer removes first on success, failure and interrupt, and the developer's - checkout must be unchanged before and after. The runner prints one JSON summary of control ids and - expected and observed codes or fixed prepack lines — never a path, frame, position, map byte or - child output — and exits 1 on any mismatch. It runs in `pnpm run ci` right after - `smoke:cli-package`, and as the required `map-controls` CI job: ubuntu, at most 20 minutes, read-only - permissions, no remote cache credentials, no pnpm cache, no build step and no upload, inside `ci`'s - needs with a guard that refuses every result except success (the required contexts stay `ci` and - `coverage`). That job's production is a second, independent build that is never shipped, not a - retry of the packed candidate, so ADR-0111's no-rebuild rule for a candidate is not touched. -- **The `package_test` receipt.** The seal and both receipt validators share one constant, the earlier - `compiled_package_test` is refused, and the receipt keeps version 1 and its twelve fields; - consumers and the aggregate verify identity through the shared validator. The mode is a claim, not - evidence of the map checks: what backs it is the required `ci` run at the same commit, and Steps - 9c and 9d must not read it as proof. `release.yml` runs no map controls of its own (its graph is - Step 9d's); a manual dry-run's `package_test` artifact is backed by the required `ci` run at the same - commit, and the interim guard refuses every release-workflow artifact anyway. - -The runner keeps private maps and manifests longer than the producer does, always under the staging; a -runner killed outright can leave the owner-only root behind, the producer's existing named limit. The -CI job runs on Linux only; a local run on Windows claims no mode evidence, as above. + baseline's (see deferred-tasks.md, "Two packs of one source"). Staging copies stay under the private + staging, which the producer removes first on success, failure and interrupt; a refused control + build's `dist`, which holds the bundler's maps, is removed at once. The developer's checkout is read, + never written (Git may refresh its own index's stat cache), and its identity must be unchanged + afterwards. The runner prints one JSON summary of control ids, expected and observed codes and + prepack's fixed lines with their repository-relative subjects — never an absolute path, frame, + position, map byte or raw child output — and exits 1 unless every expected control (seventeen, plus + four for each anchor-bearing chunk) ran exactly once with its expected outcome. It runs in + `pnpm run ci` right after `smoke:cli-package`, and as the required `map-controls` CI job: ubuntu, at + most 20 minutes, read-only permissions, no remote cache credentials, no pnpm cache, no build step, no + custom shell and no upload, inside `ci`'s needs with a guard that refuses every result except success + (the required contexts stay `ci` and `coverage`). That job's production is a second, independent + build that is never shipped, not a retry of the packed candidate, so ADR-0111's no-rebuild rule for a + candidate is not touched. +- **The `package_test` receipt.** The seal and the shared receipt validator (in its check and its + normalized record) use one constant, the earlier `compiled_package_test` is refused, and the receipt + keeps version 1 and its twelve fields; consumers and the aggregate verify identity through that + validator. The mode is a claim, not evidence of the map checks. The `ci` workflow at a commit + requires the controls, but a given artifact is backed only where a successful `ci` run exists at + exactly its commit: `pack-cli` uploads its artifact whatever the same run's `map-controls` job + concludes, a pull request's `ci` runs at its merge commit, not its head, and a release dispatch with + `allow_off_main` or of an intermediate commit has no `ci` run at all; a local dirty production writes + `package_test` too. Steps 9c and 9d must therefore never read the mode as proof. `release.yml` runs + no map controls of its own and its pack job keeps its `pnpm` dependency cache (public registry + packages only) until Step 9d rebuilds that graph, and the interim guard refuses every + release-workflow artifact. + +The runner keeps private maps and manifests longer than the producer does, always inside the +producer's owner-only root; a runner killed outright can leave that root behind, the producer's +existing named limit, and an interrupted runner leaves its own empty temporary directory (see +deferred-tasks.md). The CI job runs on Linux only; a local run on Windows claims no POSIX mode-bit +evidence, as above. ## Recipient custody diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index 8ee099c3..6fec109e 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -719,6 +719,42 @@ names both `produce.mjs` changes, INT-9A2A-R3-1); the two test gaps are recorded **Re-check:** add an anchor whose source the manifest maps and whose call site resolves in the source revision but that no captured frame reaches, and expect `ANCHOR_UNMAPPED`. *(nit · tools/cli-package/symbolicate.test.mjs)* +## Wave 3 Step 9a.2b — recorded low findings (2026-10-24) + +The internal rigor pass over Step 9a.2b at `77ee5d0a`, run before its first review round under the +2026-10-23 acceptance amendment, corrected its high finding, every medium one and the lows it could close +inside the increment +([checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-implementation-checkpoint--2026-10-24)). +These are recorded instead. **Trigger for every entry:** the next change to the file it names, or Step 9d +for the two release entries. + +- [ ] **The runner's cleanup and its copies' placement have no test.** *Low.* *(rigor pass, F5)* Removing + `runMapControls`' removal of its own temporary root, or moving a control's copies out of the staging, passes + every test; the real runner shows the placement, but nothing asserts it. **Re-check:** a test that runs the + runner's composition with recording ports (as `produce.test.mjs` does) and asserts every `scratch` path lies + under the staging and the temporary root is gone afterwards. *(low · tools/cli-package/map-controls-core.mjs)* +- [ ] **An interrupted runner leaves its own empty temporary directory.** *Low.* *(rigor pass, PLAT-9A2B-6 = + SEC-9a2b-5 = INT-2)* The producer's interrupt handler empties the artifact directory and exits, so + `runMapControls`' `finally` never removes `relavium-map-controls-*/artifact`; it holds nothing private. + **Re-check:** remove the runner's root from the interrupt path too (for example with `exitOnInterrupt` around + the run), with an interrupt test. *(low · tools/cli-package/map-controls-core.mjs)* +- [ ] **The runner's own self-check refusals have no test.** *Low.* *(rigor pass, CRIT-9A2B-2)* + `PACKED_CLI_MAP_CONTROL_CHECKOUT_CHANGED`, `_RESTORE`, `_NO_ANCHOR_CHUNK`, `_TARGET`, `_KIND` and `_ORIGIN` are + checked only by reading and, for the target and kind checks, by a runner mutation run. **Re-check:** unit + tests that drive each refusal through the runner's composition with stubbed ports. *(low · + tools/cli-package/map-controls-core.mjs)* +- [ ] **Release artifacts are not bound to a successful `ci` run at their commit.** *Medium, for Step 9d.* + *(rigor pass, PLAT-9A2B-2 = CLM-9A2B-4 = INT-1)* A release dispatch with `allow_off_main`, or of an intermediate + commit, writes a `package_test` receipt at a commit no `ci` run (so no map-controls run) ever covered; + packaging.md now says the mode is a claim and when it is backed, and the interim guard refuses every such + artifact. **Re-check:** Step 9d's release graph or custody validator requires a successful `ci` run (with + `map-controls`) at exactly the candidate's commit. *(medium · .github/workflows/release.yml)* +- [ ] **The release pack job keeps its `pnpm` dependency cache.** *Low, for Step 9d.* *(rigor pass, + CLM-9A2B-8)* `release.yml`'s pack job sets `cache: pnpm` while `ci.yml`'s `pack-cli` and `map-controls` refuse + it; the store holds only content-addressed public registry packages, which packaging.md permits, so no map + can enter it. **Re-check:** when Step 9d rebuilds the release graph, drop the cache or record why the release + pack keeps it. *(low · .github/workflows/release.yml)* + ## Native file identity on Windows — approved increment (2026-10-24) Found by Step 9a.2a's push CI (the advisory Windows job of run 40105286543). The maintainer approved a separate diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 4ce1cf14..5717eab5 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -9827,7 +9827,31 @@ controls job, as ruled. On this host the runner's 24 controls pass in about 30 s `smoke:cli-package` passes with the flipped receipt through the real producer and consumers, and `test:acceptance-tools` passes 911 of 911. -**Remaining gates.** The internal rigor pass (a whole-increment mutation sweep in which each control must fail -when its refusal is removed, the claims, platform and security lenses), the full `pnpm run ci`, then one NEW -complete cumulative round with no blocker, high or medium finding and no regression, together with the separate -scoped security sitting at the same head and bar, and the `map-controls` job green on the push CI. +**Internal rigor pass, 2026-10-24.** Under the rigor-before-review rule a pass covered the whole increment at +`77ee5d0a` before its first round: three mutation sweeps (the seal refusals and the flip; the runner, including +removing each production refusal a control is meant to prove and running the real runner; the CI pins) and +claims, platform, security and integration lenses with a completeness critic; the detailed record is local. It +confirmed one high finding: the re-seal-receipt control re-sealed against the sealed staging, so the staging half +of `PACKED_CLI_ALREADY_SEALED` refused for it and the artifact half it claimed to prove could be removed with the +runner still passing. Corrected: it now re-seals against a copy of the staging without its manifest. Medium +findings corrected: nothing tested the seal's default `verify` port (a seal test now runs the real check, and a new +`seal-verify` control changes a chunk during the seal's own run); the runner accepted whatever controls ran and +its exit gate was untested (it now requires the seventeen fixed controls plus four per anchor-bearing chunk, each +exactly once, through a tested `verdict` and `report`); its entry guard compared two differently resolved paths +and could exit 0 on Windows without running (the entry point is now an unconditional `map-controls.mjs` over +`map-controls-core.mjs`); the per-chunk controls were not checked to corrupt their own chunk in their own way (each +re-recorded copy must now differ from the sealed record in exactly its chunk, and a line or column shift must +change the line count as it names); the workflow pins did not pin the step shells, the job environment or several +properties' negative cases (they do now); and the contract's statement that a manual dry-run's `package_test` +artifact is backed by the required `ci` run at the same commit, recorded in the preflight table above, did not hold +for every dispatch — corrected in the contract and the workflow comments, with Step 9d's binding to a successful +`ci` run recorded in deferred-tasks.md. Lows corrected include an early refusal of a half-configured CI +environment, removing a refused control build's `dist`, tests of the already-sealed refusal's order, of a failed +receipt write, of the version's start anchor and of `verifySealedDist`'s every output and read bounds, and the +summary, checkout and already-sealed wording. Five lows are recorded in +[deferred-tasks.md](../deferred-tasks.md#wave-3-step-9a2b--recorded-low-findings-2026-10-24). The five mutants the +corrections target now fail the real runner, each on its own control or self-check. + +**Remaining gates.** The full `pnpm run ci`, then one NEW complete cumulative round with no blocker, high or medium +finding and no regression, together with the separate scoped security sitting at the same head and bar, and the +`map-controls` job green on the push CI. diff --git a/tools/ci/require-packed-cli.test.mjs b/tools/ci/require-packed-cli.test.mjs index 64c4e9dc..ae651f84 100644 --- a/tools/ci/require-packed-cli.test.mjs +++ b/tools/ci/require-packed-cli.test.mjs @@ -234,6 +234,11 @@ const mapControlsGuard = 'Require complete product map controls'; function checkMapControls(workflow) { const job = workflow.jobs['map-controls']; requiredJob(job); + // Every step runs in the default shell with only the job's own environment. + assert.equal(workflow.defaults, undefined); + assert.equal(job.defaults, undefined); + assert.equal(workflow.jobs.ci.defaults, undefined); + assert.deepEqual(job.env, { TURBO_TOKEN: '', TURBO_TEAM: '' }); assert.equal(job.needs, undefined); assert.equal(job.strategy, undefined); assert.equal(job['runs-on'], 'ubuntu-latest'); @@ -253,15 +258,16 @@ function checkMapControls(workflow) { job.steps.filter((step) => step.run).map((step) => step.run), ['pnpm install --frozen-lockfile', mapControlsCommand], ); - assert.deepEqual(job.steps.find((step) => step.run === mapControlsCommand).env, { - RELAVIUM_PACKAGE_SOURCE_SHA: '${{ github.sha }}', - }); + const controls = job.steps.find((step) => step.run === mapControlsCommand); + assert.deepEqual(Object.keys(controls).sort(), ['env', 'name', 'run']); + assert.deepEqual(controls.env, { RELAVIUM_PACKAGE_SOURCE_SHA: '${{ github.sha }}' }); // Required CI waits for it and refuses every result except success; the package aggregate does not. const ci = workflow.jobs.ci; assert.ok(ci.needs.includes('map-controls')); assert.ok(![workflow.jobs['packed-cli'].needs].flat().includes('map-controls')); const guard = ci.steps.find((step) => step.name === mapControlsGuard); assert.ok(guard, mapControlsGuard); + assert.deepEqual(Object.keys(guard).sort(), ['env', 'name', 'run']); assert.deepEqual(guard.env, { MAP_CONTROLS_RESULT: '${{ needs.map-controls.result }}' }); assert.equal( guard.run, @@ -303,6 +309,25 @@ for (const [name, mutate] of [ 'no source identity', (job) => delete job.steps.find((step) => step.run === mapControlsCommand).env, ], + [ + 'a tolerated controls step', + (job) => + (job.steps.find((step) => step.run === mapControlsCommand)['continue-on-error'] = true), + ], + [ + 'a custom shell on the controls step', + (job) => (job.steps.find((step) => step.run === mapControlsCommand).shell = 'true {0}'), + ], + ['a default shell for the job', (job) => (job.defaults = { run: { shell: 'true {0}' } })], + ['a preload in the job environment', (job) => (job.env.NODE_OPTIONS = '--import ./x.mjs')], + ['write permissions', (job) => (job.permissions = { contents: 'write' })], + ['another platform', (job) => (job['runs-on'] = 'macos-latest')], + ['a dependency', (job) => (job.needs = 'pack-cli')], + ['a matrix', (job) => (job.strategy = { matrix: { os: ['ubuntu-latest'] } })], + [ + 'pnpm setup inputs', + (job) => (job.steps.find((step) => step.uses === actions.pnpm).with = { version: '9.12.3' }), + ], ]) test(`map controls job refuses ${name}`, () => { const workflow = readWorkflow('ci'); @@ -331,6 +356,23 @@ for (const [name, mutate] of [ 'a guard that admits other results', (jobs) => (jobs.ci.steps.find((step) => step.name === mapControlsGuard).run = 'true'), ], + [ + 'a guard reading another result', + (jobs) => + (jobs.ci.steps.find((step) => step.name === mapControlsGuard).env = { + MAP_CONTROLS_RESULT: '${{ needs.packed-cli.result }}', + }), + ], + [ + 'a tolerated guard', + (jobs) => + (jobs.ci.steps.find((step) => step.name === mapControlsGuard)['continue-on-error'] = true), + ], + [ + 'a custom shell on the guard', + (jobs) => (jobs.ci.steps.find((step) => step.name === mapControlsGuard).shell = 'true {0}'), + ], + ['a default shell for ci', (jobs) => (jobs.ci.defaults = { run: { shell: 'true {0}' } })], ]) test(`required CI refuses ${name} of the map controls`, () => { const workflow = readWorkflow('ci'); @@ -338,6 +380,12 @@ for (const [name, mutate] of [ assert.throws(() => checkMapControls(workflow), assert.AssertionError); }); +test('a workflow-wide default shell refuses the map controls', () => { + const workflow = readWorkflow('ci'); + workflow.defaults = { run: { shell: 'true {0}' } }; + assert.throws(() => checkMapControls(workflow), assert.AssertionError); +}); + test('required CI joins packed acceptance and explicitly refuses every result except success', () => { const job = readWorkflow('ci').jobs.ci; assert.ok(job.needs.includes('packed-cli')); diff --git a/tools/ci/require-w8-acceptance.test.mjs b/tools/ci/require-w8-acceptance.test.mjs index de525fd2..77c01ac9 100644 --- a/tools/ci/require-w8-acceptance.test.mjs +++ b/tools/ci/require-w8-acceptance.test.mjs @@ -139,6 +139,7 @@ test('the actual required workflow joins both full offline controls and checks t 'node tools/cli-package/map-controls.mjs', ); const chain = packageJson.scripts.ci.split(' && '); + assert.ok(chain.indexOf('pnpm smoke:cli-package') >= 0); assert.equal( chain.indexOf('pnpm check:map-controls'), chain.indexOf('pnpm smoke:cli-package') + 1, diff --git a/tools/cli-package/README.md b/tools/cli-package/README.md index 7ebb690a..0116717c 100644 --- a/tools/cli-package/README.md +++ b/tools/cli-package/README.md @@ -13,6 +13,7 @@ build an owned map-bearing candidate and finalize it before packing. pnpm package:cli:pack /absolute/empty/artifact-directory pnpm package:cli:consume /absolute/artifact-directory /absolute/fresh/report-directory pnpm smoke:cli-package +pnpm check:map-controls ``` The producer (`produce.mjs`, run by `pack.mjs`) never builds in the developer's workspace. @@ -69,12 +70,14 @@ native addons in its own process, and needs Node 22.15 or later; an older produc the matching source revision (`symbolicate.mjs`). The manifest is discarded with the staging, and the capture's home, project and frames with the private root. The receipt does **not** certify encrypted custody or a production release candidate (Steps 9b–9d), -and its `package_test` mode is a claim, backed by the required `ci` run at the same commit. A sealed -candidate is never sealed again (`PACKED_CLI_ALREADY_SEALED`), and `dist` and the tarball must still be -the manifest's when the receipt is written (`verifySealedDist`). `pnpm check:map-controls` -(`map-controls.mjs`) runs every named negative control on product copies of one real production and -prints one JSON summary of codes, never a path or map byte; the required `map-controls` CI job runs it -uncached and uploads nothing. No private map, +and its `package_test` mode is a claim, never evidence (packaging.md says when a `ci` run backs it). An +artifact directory with a receipt, or a staging with a manifest, is never sealed again +(`PACKED_CLI_ALREADY_SEALED`), and `dist` and the tarball must still be the manifest's when the receipt +is written (`verifySealedDist`). `pnpm check:map-controls` (`map-controls.mjs` over +`map-controls-core.mjs`) runs every named negative control on product copies of one real production, +prints one JSON summary of codes and fixed lines (no absolute path, frame or map byte) and exits 1 +unless every expected control ran once with its expected outcome; the required `map-controls` CI job +runs it uncached and uploads nothing. No private map, metafile, manifest or per-source inventory enters the public artifact. The release workflow's `release-guard.mjs` refuses every artifact until verified release custody exists: it sets its exit code to 1 before it evaluates its rule (`release-guard-core.mjs`), so a rule diff --git a/tools/cli-package/binding.test.mjs b/tools/cli-package/binding.test.mjs index dec18225..def5fc98 100644 --- a/tools/cli-package/binding.test.mjs +++ b/tools/cli-package/binding.test.mjs @@ -14,6 +14,7 @@ import { REQUIRED_CONSUMER_LEGS, requirePackageJobResults, validateConsumerRecord, + isPackageVersion, validatePackageReceipt, } from './binding.mjs'; import { REQUIRED_SMOKE_CASES } from './smoke/cases.mjs'; @@ -108,6 +109,13 @@ test('one actual artifact binds every required OS/minimum-runtime consumer', () assert.equal(result.tarballSha256, digest); }); +test('a package version is a SemVer core from its first character', () => { + assert.equal(isPackageVersion('0.1.1'), true); + for (const version of ['v0.1.1', 'x0.1.1', '../0.1.1', 'relavium-0.1.1', ' 0.1.1', '0.1.1 ']) { + assert.equal(isPackageVersion(version), false, version); + } +}); + test('strict receipt refuses omitted or added private/custody claims', () => { for (const key of Object.keys(receipt())) refuses(() => validatePackageReceipt(omit(receipt(), key), expected)); diff --git a/tools/cli-package/manifest.test.mjs b/tools/cli-package/manifest.test.mjs index 2dbe15a2..93a42cef 100644 --- a/tools/cli-package/manifest.test.mjs +++ b/tools/cli-package/manifest.test.mjs @@ -467,6 +467,60 @@ test('the dist and the tarball are verified against the manifest just sealed', a } }); +test('every output is checked, each read is bounded, and so is the tarball', async () => { + // A same-length change in any output refuses, the first as much as the last. + const sealed = await sealedWithTarball(); + assert.ok(sealed.manifest.outputs.length >= 2); + for (const output of sealed.manifest.outputs) { + const path = join(sealed.fixture.cli, ...output.path.split('/')); + const saved = readFileSync(path); + const changed = Buffer.from(saved); + changed[changed.length - 1] = changed[changed.length - 1] === 0x0a ? 0x20 : 0x0a; + writeFileSync(path, changed); + assert.throws( + () => + verifySealedDist({ + candidate: sealed.fixture.root, + manifest: sealed.manifest, + tarball: sealed.tarball, + }), + refuses('DIST_DRIFT'), + output.path, + ); + writeFileSync(path, saved); + } + // An output over 8 MiB, or a tarball over 16 MiB, refuses even when the manifest records its digest. + const big = Buffer.alloc(8 * 1024 * 1024 + 1, 0x20); + const output = sealed.manifest.outputs[0]; + writeFileSync(join(sealed.fixture.cli, ...output.path.split('/')), big); + assert.throws( + () => + verifySealedDist({ + candidate: sealed.fixture.root, + manifest: { + ...sealed.manifest, + outputs: sealed.manifest.outputs.map((entry) => + entry === output ? { ...entry, sha256: digest(big) } : entry, + ), + }, + tarball: sealed.tarball, + }), + refuses('DIST_DRIFT'), + ); + const fresh = await sealedWithTarball(); + const bigger = Buffer.alloc(16 * 1024 * 1024 + 1, 0x20); + writeFileSync(fresh.tarball, bigger); + assert.throws( + () => + verifySealedDist({ + candidate: fresh.fixture.root, + manifest: { ...fresh.manifest, tarballSha256: digest(bigger) }, + tarball: fresh.tarball, + }), + refuses('TARBALL_CHANGED'), + ); +}); + test('a refusal is a ManifestError naming only its fixed code', () => { const empty = fresh('no-manifest'); mkdirSync(empty); diff --git a/tools/cli-package/map-controls-core.mjs b/tools/cli-package/map-controls-core.mjs new file mode 100644 index 00000000..5881c74f --- /dev/null +++ b/tools/cli-package/map-controls-core.mjs @@ -0,0 +1,826 @@ +/** + * Product map controls (ADR-0111; packaging.md "Product map controls and the package-test receipt — + * Step 9a.2b"; the 2026-10-24 preflight rulings), run by map-controls.mjs. One real package-test + * production runs at the checked-out commit. Inside its seal, after the baseline receipt and while the + * private staging still exists, every named negative control corrupts one product copy — the + * producer's own owned candidate tree, or a re-recorded copy of its private staging — and requires that + * control's exact refusal; each corrupted file is restored and checked by digest, and a restoration + * production must then seal again. Staging copies stay under the staging, which the producer removes + * first on success, failure and interrupt; a refused control build's `dist`, which holds the bundler's + * maps, is removed at once. The developer's checkout is read, never written (Git may refresh its own + * index's stat cache). The summary carries control ids, expected and observed codes and prepack's fixed + * lines with their repository-relative subjects — never an absolute path, frame, position, map byte or + * raw child output. + */ +import assert from 'node:assert/strict'; +import { Buffer } from 'node:buffer'; +import { + chmodSync, + cpSync, + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readdirSync, + realpathSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, dirname, join } from 'node:path'; +import { originFromEnvironment } from './binding.mjs'; +import { sourceIdentity } from './candidate.mjs'; +import { CANDIDATE_STAGING_ENV } from './finalize.mjs'; +import { readOrdinaryFile, sha256 } from './files.mjs'; +import { readPrivateManifest, sealPrivateManifest, verifySealedDist } from './manifest.mjs'; +import { emitFreshPackage } from './pipeline.mjs'; +import { producePackage } from './produce.mjs'; +import { productionFailure } from './production-failure.mjs'; +import { sealPackage } from './seal.mjs'; +import { + assertGeneratedRange, + assertOriginalRange, + decodeMappings, + lineLengths, +} from './source-map.mjs'; +import { ANCHORS, symbolicateCandidate } from './symbolicate.mjs'; + +const BASE64 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; +const MIB = 1024 * 1024; + +/** Base64 VLQ of one signed value, as a source map writer encodes it. */ +function vlq(value) { + let rest = value < 0 ? -value * 2 + 1 : value * 2; + let out = ''; + do { + let digit = rest % 32; + rest = Math.floor(rest / 32); + if (rest > 0) digit += 32; + out += BASE64[digit]; + } while (rest > 0); + return out; +} + +/** + * Encode decoded segments (as `decodeMappings` returns them) back into `mappings`. + * @param {import('./source-map.mjs').Segment[][]} lines + */ +export function encodeMappings(lines) { + let source = 0; + let originalLine = 0; + let originalColumn = 0; + let name = 0; + return lines + .map((segments) => { + let column = 0; + return segments + .map((segment) => { + let out = vlq(segment.generatedColumn - column); + column = segment.generatedColumn; + if (segment.source !== undefined) { + out += + vlq(segment.source - source) + + vlq(segment.originalLine - originalLine) + + vlq(segment.originalColumn - originalColumn); + source = segment.source; + originalLine = segment.originalLine; + originalColumn = segment.originalColumn; + if (segment.name !== undefined) { + out += vlq(segment.name - name); + name = segment.name; + } + } + return out; + }) + .join(','); + }) + .join(';'); +} + +/** + * A map shifted while staying valid: every original column after a line's start moves one to the left, + * so every position stays inside its text and only symbolication can tell. + * @param {string} mappings @param {{sources: number, names: number}} counts + */ +export function shiftOriginalColumns(mappings, counts) { + const lines = decodeMappings(mappings, counts); + let moved = 0; + for (const segments of lines) { + for (const segment of segments) { + if (segment.originalColumn !== undefined && segment.originalColumn > 0) { + segment.originalColumn -= 1; + moved += 1; + } + } + } + assert.ok(moved > 0, 'PACKED_CLI_MAP_CONTROL_NOTHING_SHIFTED'); + return encodeMappings(lines); +} + +/** Sources rotated by one: with two or more distinct sources, no source keeps its index. */ +export function rotateSources(sources) { + assert.ok( + sources.length >= 2 && new Set(sources).size === sources.length, + 'PACKED_CLI_MAP_CONTROL_SOURCES', + ); + return [...sources.slice(1), sources[0]]; +} + +/** The JS one line down: a line break after the hashbang line, or at the start. */ +export function shiftLines(text) { + if (!text.startsWith('#!')) return `\n${text}`; + const end = text.indexOf('\n'); + assert.ok(end > 0, 'PACKED_CLI_MAP_CONTROL_HASHBANG'); + return `${text.slice(0, end + 1)}\n${text.slice(end + 1)}`; +} + +/** The JS one column right: a space before every line but a hashbang line. */ +export function shiftColumns(text) { + return text + .split('\n') + .map((line, index) => (index === 0 && line.startsWith('#!') ? line : ` ${line}`)) + .join('\n'); +} + +/** `mappings` cut right after the first continuation digit, so the value it starts never ends. */ +export function cutInsideValue(mappings) { + for (let index = 0; index < mappings.length; index += 1) { + if (BASE64.indexOf(mappings[index]) >= 32) return mappings.slice(0, index + 1); + } + assert.fail('PACKED_CLI_MAP_CONTROL_NO_CONTINUATION'); +} + +/** An owner-only directory, created fresh. */ +function privateDirectory(path) { + mkdirSync(path, { recursive: true, mode: 0o700 }); + chmodSync(path, 0o700); + return path; +} + +/** The refusal of `action`: its code, its prepack diagnostics and how many times stdout says each marker. */ +async function refusal(action) { + try { + await action(); + } catch (error) { + const lines = productionFailure(error); + const stdout = typeof error?.result?.stdout === 'string' ? error.result.stdout : ''; + return { + code: /: ([A-Z0-9_]+)$/u.exec(lines.at(-1))?.[1] ?? 'UNEXPECTED', + diagnostics: lines.slice(0, -1), + builds: (stdout.match(/\bESM Build start\b/gu) ?? []).length, + finalized: (stdout.match(/^✓ CLI candidate finalized: /gmu) ?? []).length, + }; + } + return { code: 'ACCEPTED', diagnostics: [], builds: 0, finalized: 0 }; +} + +/** A file's bytes and mode, to restore it exactly. */ +function snapshot(path) { + const bytes = readOrdinaryFile(path, 16 * MIB); + return { path, bytes, sha256: sha256(bytes), mode: lstatSync(path).mode & 0o777 }; +} + +/** Put a snapshot back and prove it is byte for byte the original. */ +function restore(saved) { + rmSync(saved.path, { force: true }); + writeFileSync(saved.path, saved.bytes, { mode: saved.mode, flag: 'wx' }); + chmodSync(saved.path, saved.mode); + assert.equal( + sha256(readOrdinaryFile(saved.path, 16 * MIB)), + saved.sha256, + 'PACKED_CLI_MAP_CONTROL_RESTORE', + ); +} + +/** SHA-256 of every file in a directory, by name; a directory listed as such. */ +function inventory(directory, skip = new Set()) { + return Object.fromEntries( + readdirSync(directory, { withFileTypes: true, recursive: true }) + .map((entry) => join(entry.parentPath ?? entry.path, entry.name)) + .filter((path) => ![...skip].some((name) => path.startsWith(join(directory, name)))) + .sort() + .map((path) => [ + path.slice(directory.length), + lstatSync(path).isFile() ? sha256(readOrdinaryFile(path, 32 * MIB)) : 'directory', + ]), + ); +} + +/** + * Run every control inside the baseline seal. `context` holds the seal's own inputs, its receipt, the + * baseline artifact directory and the emitter's isolated environment and pnpm. + * @returns {Promise<{id: string, expected: string, observed: string}[]>} + */ +async function runControls(context) { + const { candidate, staging, checkout, source, dirty, receipt, artifact, env, pnpm } = context; + const results = []; + const record = (id, expected, observed) => results.push({ id, expected, observed }); + const controls = privateDirectory(join(staging, 'controls')); + let count = 0; + const scratch = (id) => + privateDirectory(join(controls, `${String((count += 1)).padStart(2, '0')}-${id}`)); + const cli = join(candidate, 'apps', 'cli'); + const tarball = join(artifact, receipt.tarball); + const sealed = readPrivateManifest(staging); + const distDigests = () => inventory(join(cli, 'dist')); + + // The chunks the anchors' sources lie in, by their staged maps; the anchor sources' upstream map. + const anchorSources = new Set(ANCHORS.map((anchor) => anchor.source)); + const mapsOf = (name) => + JSON.parse(readOrdinaryFile(join(staging, 'maps', name), 16 * MIB).toString('utf8')); + const chunks = readdirSync(join(staging, 'maps')) + .filter((name) => mapsOf(name).sources.some((path) => anchorSources.has(path))) + .map((name) => name.slice(0, -'.map'.length)) + .sort(); + assert.ok(chunks.length > 0, 'PACKED_CLI_MAP_CONTROL_NO_ANCHOR_CHUNK'); + const plain = sealed.outputs + .map((output) => basename(output.path)) + .filter((name) => !chunks.includes(name)) + .sort(); + + const sealedRecord = JSON.parse( + readOrdinaryFile(join(staging, 'finalized.json'), MIB).toString('utf8'), + ); + const sealedDigests = new Map( + [...sealedRecord.outputs, ...sealedRecord.maps].map((entry) => [entry.path, entry.sha256]), + ); + + /** + * A copy of the sealed staging re-recorded for the candidate as it now is, then symbolicated. The + * copy must differ from the sealed record in exactly the `changed` outputs and maps, so a control + * corrupts the chunk it names and nothing else. + */ + const resealed = async (id, { forceDirty = dirty, changeMap, changed = [] } = {}) => { + const root = scratch(id); + const copy = privateDirectory(join(root, 'staging')); + privateDirectory(join(copy, 'maps')); + const finalized = JSON.parse( + readOrdinaryFile(join(staging, 'finalized.json'), MIB).toString('utf8'), + ); + finalized.maps = finalized.maps.map((entry) => { + const name = basename(entry.path); + let bytes = readOrdinaryFile(join(staging, 'maps', name), 16 * MIB); + if (changeMap !== undefined && name === `${changeMap.chunk}.map`) { + bytes = Buffer.from(JSON.stringify(changeMap.change(JSON.parse(bytes.toString('utf8'))))); + } + writeFileSync(join(copy, 'maps', name), bytes, { mode: 0o600, flag: 'wx' }); + return { path: entry.path, bytes: bytes.length, sha256: sha256(bytes) }; + }); + finalized.outputs = finalized.outputs.map((entry) => { + const bytes = readOrdinaryFile(join(cli, ...entry.path.split('/')), 8 * MIB); + return { path: entry.path, bytes: bytes.length, sha256: sha256(bytes) }; + }); + assert.deepEqual( + [...finalized.outputs, ...finalized.maps] + .filter((entry) => sealedDigests.get(entry.path) !== entry.sha256) + .map((entry) => entry.path) + .sort(), + [...changed].sort(), + 'PACKED_CLI_MAP_CONTROL_TARGET', + ); + writeFileSync(join(copy, 'finalized.json'), JSON.stringify(finalized), { + mode: 0o600, + flag: 'wx', + }); + const digests = new Map( + finalized.outputs.map((output) => [`package/${output.path}`, output.sha256]), + ); + const archive = { + sha256: receipt.tarballSha256, + package: receipt.package, + files: receipt.files.map((file) => + digests.has(file.path) ? { ...file, sha256: digests.get(file.path) } : file, + ), + }; + await sealPrivateManifest({ + candidate, + staging: copy, + checkout, + source, + dirty: forceDirty, + archive, + }); + const work = privateDirectory(join(root, 'work')); + return refusal(() => + symbolicateCandidate({ candidate, staging: copy, checkout, work, archive }), + ); + }; + + /** One pack of the candidate as it now is, into a fresh staging and artifact directory. */ + const pack = async (id, stagingOverride) => { + const root = scratch(id); + const destination = privateDirectory(join(root, 'artifact')); + const fresh = stagingOverride ?? privateDirectory(join(root, 'staging')); + const observed = await refusal(() => + pnpm(['pack', '--pack-destination', destination], { + cwd: cli, + env: { ...env, [CANDIDATE_STAGING_ENV]: fresh }, + }), + ); + return { observed, destination, staging: fresh, root }; + }; + + /** Seal a fresh pack's tarball with fresh work, as the producer would. */ + const seal = (packed) => + refusal(() => + sealPackage(packed.destination, { + repository: candidate, + staging: packed.staging, + source, + dirty, + checkout, + work: privateDirectory(join(packed.root, 'work')), + }), + ); + + // Post-receipt rebuild refusals that need the sealed state come first. + { + // A second prepack into the sealed staging refuses before any build, changing nothing. + const before = { dist: distDigests(), staging: inventory(staging, new Set(['controls'])) }; + const second = await pack('second-prepack', staging); + const unchanged = + JSON.stringify(distDigests()) === JSON.stringify(before.dist) && + JSON.stringify(inventory(staging, new Set(['controls']))) === + JSON.stringify(before.staging) && + readdirSync(second.destination).length === 0 && + second.observed.builds === 0; + record( + 'second-prepack', + 'STAGING_NOT_EMPTY', + unchanged && + second.observed.diagnostics.length === 1 && + second.observed.diagnostics[0] === '✗ CLI candidate finalization refused: STAGING_NOT_EMPTY' + ? 'STAGING_NOT_EMPTY' + : `MARKERS_${second.observed.code}`, + ); + } + { + // A re-seal into the artifact directory, or of the sealed staging beside a copied tarball, refuses + // before any capture runs. + const receiptDigest = sha256( + readOrdinaryFile(join(artifact, 'package-test-receipt.json'), MIB), + ); + const root = scratch('re-seal'); + const works = [privateDirectory(join(root, 'work-1')), privateDirectory(join(root, 'work-2'))]; + // The receipt half alone: a copy of the sealed staging without its manifest. + const unsealed = privateDirectory(join(root, 'unsealed')); + cpSync(join(staging, 'maps'), join(unsealed, 'maps'), { recursive: true }); + for (const name of ['finalized.json', 'metafile-esm.json']) { + cpSync(join(staging, name), join(unsealed, name)); + } + const again = await refusal(() => + sealPackage(artifact, { + repository: candidate, + staging: unsealed, + source, + dirty, + checkout, + work: works[0], + }), + ); + // The staging half alone: the tarball copied without its receipt, beside the sealed staging. + const copied = privateDirectory(join(root, 'artifact')); + cpSync(tarball, join(copied, receipt.tarball)); + const beside = await refusal(() => + sealPackage(copied, { + repository: candidate, + staging, + source, + dirty, + checkout, + work: works[1], + }), + ); + const quiet = + works.every((work) => readdirSync(work).length === 0) && + !existsSync(join(unsealed, 'manifest.json')) && + !existsSync(join(copied, 'package-test-receipt.json')) && + sha256(readOrdinaryFile(join(artifact, 'package-test-receipt.json'), MIB)) === receiptDigest; + record( + 're-seal-receipt', + 'PACKED_CLI_ALREADY_SEALED', + quiet ? again.code : `MARKERS_${again.code}`, + ); + record( + 're-seal-staging', + 'PACKED_CLI_ALREADY_SEALED', + quiet ? beside.code : `MARKERS_${beside.code}`, + ); + } + + // The re-recording technique alone refuses nothing: the unchanged copy reaches every anchor. + record('re-recorded-baseline', 'ACCEPTED', (await resealed('re-recorded-baseline')).code); + + // Wrong JS by line and by column, a wrong map and permuted sources, on every anchor-bearing chunk. + for (const [index, chunk] of chunks.entries()) { + const path = join(cli, 'dist', chunk); + for (const [kind, change] of [ + ['js-line', shiftLines], + ['js-column', shiftColumns], + ]) { + const saved = snapshot(path); + try { + const text = saved.bytes.toString('utf8'); + const changedText = change(text); + // One more line for a line shift, the same lines for a column shift. + assert.ok( + changedText !== text && + changedText.split('\n').length - text.split('\n').length === + (kind === 'js-line' ? 1 : 0), + 'PACKED_CLI_MAP_CONTROL_KIND', + ); + writeFileSync(path, changedText); + record( + `${kind}-${index + 1}`, + 'PACKED_CLI_SYMBOLICATION_ANCHOR_UNMAPPED', + (await resealed(kind, { changed: [`dist/${chunk}`] })).code, + ); + } finally { + restore(saved); + } + } + record( + `map-line-${index + 1}`, + 'PACKED_CLI_SYMBOLICATION_ANCHOR_UNMAPPED', + ( + await resealed('map-line', { + changeMap: { chunk, change: (map) => ({ ...map, mappings: `;${map.mappings}` }) }, + changed: [`dist/${chunk}.map`], + }) + ).code, + ); + record( + `sources-rotated-${index + 1}`, + 'PACKED_CLI_SYMBOLICATION_ANCHOR_UNMAPPED', + ( + await resealed('sources-rotated', { + changeMap: { chunk, change: (map) => ({ ...map, sources: rotateSources(map.sources) }) }, + changed: [`dist/${chunk}.map`], + }) + ).code, + ); + } + + // A wrong source revision: clean, the Git object refuses the copy; dirty, the anchor positions move. + { + const saved = snapshot(join(candidate, ...ANCHORS[1].source.split('/'))); + try { + writeFileSync(saved.path, `// shifted\n${saved.bytes.toString('utf8')}`); + record( + 'source-clean', + 'PACKED_CLI_SYMBOLICATION_SOURCE_REVISION', + (await resealed('source-clean', { forceDirty: false })).code, + ); + record( + 'source-dirty', + 'PACKED_CLI_SYMBOLICATION_ANCHOR_UNMAPPED', + (await resealed('source-dirty', { forceDirty: true })).code, + ); + } finally { + restore(saved); + } + } + + // A rebuilt dist that differs from the manifest: the sealed state verifies; a changed chunk whose map + // names no anchor source, and a real rebuild from a changed source, refuse. + record( + 'sealed-dist', + 'ACCEPTED', + (await refusal(() => verifySealedDist({ candidate, manifest: sealed, tarball }))).code, + ); + if (plain.length === 0) + record('dist-plain-chunk', 'PACKED_CLI_MANIFEST_DIST_DRIFT', 'NO_PLAIN_CHUNK'); + else { + const saved = snapshot(join(cli, 'dist', plain[0])); + try { + writeFileSync(saved.path, Buffer.concat([saved.bytes, Buffer.from(' ')])); + record( + 'dist-plain-chunk', + 'PACKED_CLI_MANIFEST_DIST_DRIFT', + (await refusal(() => verifySealedDist({ candidate, manifest: sealed, tarball }))).code, + ); + } finally { + restore(saved); + } + } + const entry = snapshot(join(cli, 'src', 'index.ts')); + try { + writeFileSync( + entry.path, + `${entry.bytes.toString('utf8')}\nglobalThis.relaviumMapControls = 1;\n`, + ); + const rebuilt = await pack('rebuild'); + const changed = sealed.outputs.some((output) => { + try { + return ( + sha256(readOrdinaryFile(join(cli, ...output.path.split('/')), 8 * MIB)) !== output.sha256 + ); + } catch { + return true; + } + }); + const verified = await refusal(() => + verifySealedDist({ candidate, manifest: sealed, tarball }), + ); + record( + 'dist-rebuilt', + 'PACKED_CLI_MANIFEST_DIST_DRIFT', + rebuilt.observed.code === 'ACCEPTED' && changed + ? verified.code + : `MARKERS_${rebuilt.observed.code}`, + ); + } finally { + restore(entry); + } + + // Upstream maps: a missing, a malformed and a value-truncated map refuse in the finalizer after the + // bundler silently succeeded; a byte-truncated map fails the build; a shifted but valid map is + // accepted there and refused by symbolication. + const upstream = ANCHORS[0].source + .replace(/^packages\/core\/src\//u, 'packages/core/dist/') + .replace(/\.ts$/u, '.js'); + const upstreamMap = join(candidate, ...`${upstream}.map`.split('/')); + const saved = snapshot(upstreamMap); + const original = JSON.parse(saved.bytes.toString('utf8')); + const refusedLine = (code, subject) => + `✗ CLI candidate finalization refused: ${code} (${subject})`; + for (const [id, corrupt, expected] of [ + ['upstream-missing', () => rmSync(upstreamMap), refusedLine('UPSTREAM_MAP_MISSING', upstream)], + [ + 'upstream-shape', + () => writeFileSync(upstreamMap, JSON.stringify({ ...original, sourcesContent: [null] })), + refusedLine('UPSTREAM_MAP_SHAPE', `${upstream}.map`), + ], + [ + 'upstream-index', + () => + writeFileSync( + upstreamMap, + JSON.stringify({ ...original, mappings: `A${vlq(original.sources.length)}AA` }), + ), + refusedLine('UPSTREAM_MAP_MAPPINGS', `${upstream}.map`), + ], + [ + // A map that is no longer JSON fails the bundler's own build, before the finalizer reads it. + 'upstream-cut-bytes', + () => writeFileSync(upstreamMap, saved.bytes.subarray(0, Math.floor(saved.bytes.length / 2))), + '✗ CLI build failed', + ], + [ + 'upstream-cut-value', + () => + writeFileSync( + upstreamMap, + JSON.stringify({ ...original, mappings: cutInsideValue(original.mappings) }), + ), + refusedLine('UPSTREAM_MAP_MAPPINGS', `${upstream}.map`), + ], + ]) { + try { + corrupt(); + const packed = await pack(id); + // The bundler silently succeeded once, nothing was finalized, staged or packed, and the one + // prepack diagnostic is the expected fixed line. + const markers = [ + packed.observed.builds, + packed.observed.finalized, + readdirSync(packed.staging).length, + readdirSync(packed.destination).length, + packed.observed.diagnostics.length, + ]; + const line = packed.observed.diagnostics.at(-1); + record( + id, + expected, + markers.join() === '1,0,0,0,1' && line === expected + ? expected + : `MARKERS(${markers.join()}) ${line ?? packed.observed.code}`, + ); + } finally { + restore(saved); + // The refused build left the bundler's own maps in dist: none outlives the control. + rmSync(join(cli, 'dist'), { recursive: true, force: true }); + } + } + try { + const mappings = shiftOriginalColumns(original.mappings, { + sources: original.sources.length, + names: original.names.length, + }); + // The shift is strictly valid: the finalizer's own checks admit it. + const lines = decodeMappings(mappings, { + sources: original.sources.length, + names: original.names.length, + }); + assertGeneratedRange( + lines, + lineLengths( + readOrdinaryFile(join(candidate, ...upstream.split('/')), 16 * MIB).toString('utf8'), + ), + ); + assertOriginalRange( + lines, + original.sources.map((path) => + lineLengths( + readOrdinaryFile(join(dirname(upstreamMap), ...path.split('/')), 16 * MIB).toString( + 'utf8', + ), + ), + ), + ); + writeFileSync(upstreamMap, JSON.stringify({ ...original, mappings })); + const packed = await pack('upstream-shifted'); + const accepted = packed.observed.code === 'ACCEPTED'; + record( + 'upstream-shifted', + 'PACKED_CLI_SYMBOLICATION_ANCHOR_UNMAPPED', + accepted ? (await seal(packed)).code : `MARKERS_${packed.observed.code}`, + ); + } finally { + restore(saved); + } + + // The seal's own check: a dist the run changed after the inspection refuses before the receipt. + { + const packed = await pack('seal-verify'); + let observed = `MARKERS_${packed.observed.code}`; + if (packed.observed.code === 'ACCEPTED' && plain.length > 0) { + const refused = await refusal(() => + sealPackage(packed.destination, { + repository: candidate, + staging: packed.staging, + source, + dirty, + checkout, + work: privateDirectory(join(packed.root, 'work')), + ports: { + symbolicate: async (options) => { + await symbolicateCandidate(options); + const path = join(cli, 'dist', plain[0]); + writeFileSync( + path, + Buffer.concat([readOrdinaryFile(path, 8 * MIB), Buffer.from(' ')]), + ); + }, + }, + }), + ); + observed = existsSync(join(packed.destination, 'package-test-receipt.json')) + ? 'MARKERS_RECEIPT_WRITTEN' + : refused.code; + } + record('seal-verify', 'PACKED_CLI_MANIFEST_DIST_DRIFT', observed); + } + + // Restoration: every corrupted file is back, so a fresh pack seals again. + const restored = await pack('restoration'); + record( + 'restoration', + 'ACCEPTED', + restored.observed.code === 'ACCEPTED' + ? (await seal(restored)).code + : `MARKERS_${restored.observed.code}`, + ); + return { results, chunks: chunks.length }; +} + +/** + * Run the controls through one real production of `repository` and return their results with the + * source identity. The developer checkout must be the same before and after. + * @param {{repository: string, environment: Record, + * temporary?: string}} options + */ +export async function runMapControls({ repository, environment, temporary = tmpdir() }) { + // A half-configured CI environment refuses before anything is built. + try { + originFromEnvironment(environment); + } catch { + assert.fail('PACKED_CLI_MAP_CONTROL_ORIGIN'); + } + const before = await sourceIdentity(repository); + const root = realpathSync.native(mkdtempSync(join(temporary, 'relavium-map-controls-'))); + chmodSync(root, 0o700); + let ran = { results: [], chunks: 0 }; + let emitted; + try { + const produced = await producePackage({ + repository, + destination: join(root, 'artifact'), + environment, + temporary, + ports: { + emit: (options) => { + emitted = { env: options.env, pnpm: options.pnpm }; + return emitFreshPackage(options); + }, + seal: async (artifact, inputs) => { + const receipt = await sealPackage(artifact, inputs); + ran = await runControls({ + candidate: inputs.repository, + staging: inputs.staging, + checkout: inputs.checkout, + source: inputs.source, + dirty: inputs.dirty, + receipt, + artifact, + ...emitted, + }); + return receipt; + }, + }, + }); + const after = await sourceIdentity(repository); + assert.deepEqual( + [after.source, after.status, after.unchecked], + [before.source, before.status, before.unchecked], + 'PACKED_CLI_MAP_CONTROL_CHECKOUT_CHANGED', + ); + return { source: produced.receipt.source, dirty: produced.identity.dirty, ...ran }; + } finally { + rmSync(root, { recursive: true, force: true }); + } +} + +/** The controls every run records once, besides four for each anchor-bearing chunk. */ +export const FIXED_CONTROLS = Object.freeze([ + 'second-prepack', + 're-seal-receipt', + 're-seal-staging', + 're-recorded-baseline', + 'source-clean', + 'source-dirty', + 'sealed-dist', + 'dist-plain-chunk', + 'dist-rebuilt', + 'upstream-missing', + 'upstream-shape', + 'upstream-index', + 'upstream-cut-bytes', + 'upstream-cut-value', + 'upstream-shifted', + 'seal-verify', + 'restoration', +]); +const PER_CHUNK = Object.freeze(['js-line', 'js-column', 'map-line', 'sources-rotated']); + +/** + * The verdict on a run: at least one anchor-bearing chunk, every expected control recorded exactly + * once, and each with its expected outcome. + * @param {{results: {id: string, expected: string, observed: string}[], chunks: number}} outcome + */ +export function verdict({ results, chunks }) { + const expected = [ + ...FIXED_CONTROLS, + ...Array.from({ length: chunks }, (_, index) => + PER_CHUNK.map((kind) => `${kind}-${index + 1}`), + ).flat(), + ]; + const ids = results.map((result) => result.id); + const missing = expected.filter((id) => !ids.includes(id)); + const unexpected = ids.filter((id, index) => !expected.includes(id) || ids.indexOf(id) !== index); + const failed = results + .filter((result) => result.observed !== result.expected) + .map((result) => result.id); + return { + expected: expected.length, + missing, + unexpected, + failed, + passed: chunks >= 1 && missing.length === 0 && unexpected.length === 0 && failed.length === 0, + }; +} + +/** + * Print a run's summary and return the process exit code: 0 only when the verdict passes. + * @param {{source: string, dirty: boolean, results: {id: string, expected: string, + * observed: string}[], chunks: number}} outcome + * @param {{seconds: number, write?: (line: string) => void, warn?: (line: string) => void}} options + */ +export function report(outcome, { seconds, write = console.log, warn = console.error }) { + const judged = verdict(outcome); + write( + JSON.stringify({ + scope: 'product copies of one real package-test production', + source: outcome.source, + dirty: outcome.dirty, + chunks: outcome.chunks, + controls: outcome.results.length, + expected: judged.expected, + failed: judged.failed, + missing: judged.missing, + unexpected: judged.unexpected, + results: outcome.results, + seconds, + }), + ); + if (judged.passed) return 0; + const named = [ + ...judged.failed, + ...judged.missing.map((id) => `${id} (missing)`), + ...judged.unexpected.map((id) => `${id} (unexpected)`), + ]; + warn( + `✗ CLI product map controls failed: ${named.length > 0 ? named.join(', ') : 'no anchor-bearing chunk'}`, + ); + return 1; +} diff --git a/tools/cli-package/map-controls.mjs b/tools/cli-package/map-controls.mjs index b6eecfc0..7c4a112c 100644 --- a/tools/cli-package/map-controls.mjs +++ b/tools/cli-package/map-controls.mjs @@ -1,692 +1,22 @@ /** - * Product map controls (ADR-0111; packaging.md "Product map controls — Step 9a.2b"; the 2026-10-24 - * preflight rulings). One real package-test production runs at the checked-out commit. Inside its - * seal, after the baseline receipt and while the private staging still exists, every named negative - * control corrupts one product copy — the producer's own owned candidate tree, or a copy of its private - * staging — and requires that control's exact refusal; each corrupted file is restored and checked by - * digest, and a restoration production must then seal again. Every copy a control makes stays under - * the staging, which the producer removes first on success, failure and interrupt; the developer's - * checkout is never touched. One JSON summary of control ids, expected and observed codes and outcomes - * is printed — never a path, frame, position, map byte or child output. + * Command-line entry point of the product map controls (`map-controls-core.mjs`; `pnpm + * check:map-controls`): one real production at this checkout, every named control, one JSON summary. + * It runs unconditionally, so it can never exit 0 without running the controls, and exits 1 unless + * every expected control ran once with its expected outcome. A failure prints only fixed lines. */ -import assert from 'node:assert/strict'; -import { Buffer } from 'node:buffer'; -import { - chmodSync, - cpSync, - lstatSync, - mkdirSync, - mkdtempSync, - readdirSync, - realpathSync, - rmSync, - writeFileSync, -} from 'node:fs'; -import { tmpdir } from 'node:os'; -import { basename, dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { sourceIdentity } from './candidate.mjs'; -import { CANDIDATE_STAGING_ENV } from './finalize.mjs'; -import { readOrdinaryFile, sha256 } from './files.mjs'; -import { readPrivateManifest, sealPrivateManifest, verifySealedDist } from './manifest.mjs'; -import { emitFreshPackage } from './pipeline.mjs'; -import { producePackage } from './produce.mjs'; +import { report, runMapControls } from './map-controls-core.mjs'; import { productionFailure } from './production-failure.mjs'; -import { sealPackage } from './seal.mjs'; -import { - assertGeneratedRange, - assertOriginalRange, - decodeMappings, - lineLengths, -} from './source-map.mjs'; -import { ANCHORS, symbolicateCandidate } from './symbolicate.mjs'; -const BASE64 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; -const MIB = 1024 * 1024; - -/** Base64 VLQ of one signed value, as a source map writer encodes it. */ -function vlq(value) { - let rest = value < 0 ? -value * 2 + 1 : value * 2; - let out = ''; - do { - let digit = rest % 32; - rest = Math.floor(rest / 32); - if (rest > 0) digit += 32; - out += BASE64[digit]; - } while (rest > 0); - return out; -} - -/** - * Encode decoded segments (as `decodeMappings` returns them) back into `mappings`. - * @param {import('./source-map.mjs').Segment[][]} lines - */ -export function encodeMappings(lines) { - let source = 0; - let originalLine = 0; - let originalColumn = 0; - let name = 0; - return lines - .map((segments) => { - let column = 0; - return segments - .map((segment) => { - let out = vlq(segment.generatedColumn - column); - column = segment.generatedColumn; - if (segment.source !== undefined) { - out += - vlq(segment.source - source) + - vlq(segment.originalLine - originalLine) + - vlq(segment.originalColumn - originalColumn); - source = segment.source; - originalLine = segment.originalLine; - originalColumn = segment.originalColumn; - if (segment.name !== undefined) { - out += vlq(segment.name - name); - name = segment.name; - } - } - return out; - }) - .join(','); - }) - .join(';'); -} - -/** - * A map shifted while staying valid: every original column after a line's start moves one to the left, - * so every position stays inside its text and only symbolication can tell. - * @param {string} mappings @param {{sources: number, names: number}} counts - */ -export function shiftOriginalColumns(mappings, counts) { - const lines = decodeMappings(mappings, counts); - let moved = 0; - for (const segments of lines) { - for (const segment of segments) { - if (segment.originalColumn !== undefined && segment.originalColumn > 0) { - segment.originalColumn -= 1; - moved += 1; - } - } - } - assert.ok(moved > 0, 'PACKED_CLI_MAP_CONTROL_NOTHING_SHIFTED'); - return encodeMappings(lines); -} - -/** Sources rotated by one: with two or more distinct sources, no source keeps its index. */ -export function rotateSources(sources) { - assert.ok( - sources.length >= 2 && new Set(sources).size === sources.length, - 'PACKED_CLI_MAP_CONTROL_SOURCES', - ); - return [...sources.slice(1), sources[0]]; -} - -/** The JS one line down: a line break after the hashbang line, or at the start. */ -export function shiftLines(text) { - if (!text.startsWith('#!')) return `\n${text}`; - const end = text.indexOf('\n'); - assert.ok(end > 0, 'PACKED_CLI_MAP_CONTROL_HASHBANG'); - return `${text.slice(0, end + 1)}\n${text.slice(end + 1)}`; -} - -/** The JS one column right: a space before every line but a hashbang line. */ -export function shiftColumns(text) { - return text - .split('\n') - .map((line, index) => (index === 0 && line.startsWith('#!') ? line : ` ${line}`)) - .join('\n'); -} - -/** `mappings` cut right after the first continuation digit, so the value it starts never ends. */ -export function cutInsideValue(mappings) { - for (let index = 0; index < mappings.length; index += 1) { - if (BASE64.indexOf(mappings[index]) >= 32) return mappings.slice(0, index + 1); - } - assert.fail('PACKED_CLI_MAP_CONTROL_NO_CONTINUATION'); -} - -/** An owner-only directory, created fresh. */ -function privateDirectory(path) { - mkdirSync(path, { recursive: true, mode: 0o700 }); - chmodSync(path, 0o700); - return path; -} - -/** The refusal of `action`: its code, its prepack diagnostics and how many times stdout says each marker. */ -async function refusal(action) { - try { - await action(); - } catch (error) { - const lines = productionFailure(error); - const stdout = typeof error?.result?.stdout === 'string' ? error.result.stdout : ''; - return { - code: /: ([A-Z0-9_]+)$/u.exec(lines.at(-1))?.[1] ?? 'UNEXPECTED', - diagnostics: lines.slice(0, -1), - builds: (stdout.match(/\bESM Build start\b/gu) ?? []).length, - finalized: (stdout.match(/^✓ CLI candidate finalized: /gmu) ?? []).length, - }; - } - return { code: 'ACCEPTED', diagnostics: [], builds: 0, finalized: 0 }; -} - -/** A file's bytes and mode, to restore it exactly. */ -function snapshot(path) { - const bytes = readOrdinaryFile(path, 16 * MIB); - return { path, bytes, sha256: sha256(bytes), mode: lstatSync(path).mode & 0o777 }; -} - -/** Put a snapshot back and prove it is byte for byte the original. */ -function restore(saved) { - rmSync(saved.path, { force: true }); - writeFileSync(saved.path, saved.bytes, { mode: saved.mode, flag: 'wx' }); - chmodSync(saved.path, saved.mode); - assert.equal( - sha256(readOrdinaryFile(saved.path, 16 * MIB)), - saved.sha256, - 'PACKED_CLI_MAP_CONTROL_RESTORE', - ); -} - -/** SHA-256 of every file in a directory, by name; a directory listed as such. */ -function inventory(directory, skip = new Set()) { - return Object.fromEntries( - readdirSync(directory, { withFileTypes: true, recursive: true }) - .map((entry) => join(entry.parentPath ?? entry.path, entry.name)) - .filter((path) => ![...skip].some((name) => path.startsWith(join(directory, name)))) - .sort() - .map((path) => [ - path.slice(directory.length), - lstatSync(path).isFile() ? sha256(readOrdinaryFile(path, 32 * MIB)) : 'directory', - ]), - ); -} - -/** - * Run every control inside the baseline seal. `context` holds the seal's own inputs, its receipt, the - * baseline artifact directory and the emitter's isolated environment and pnpm. - * @returns {Promise<{id: string, expected: string, observed: string}[]>} - */ -async function runControls(context) { - const { candidate, staging, checkout, source, dirty, receipt, artifact, env, pnpm } = context; - const results = []; - const record = (id, expected, observed) => results.push({ id, expected, observed }); - const controls = privateDirectory(join(staging, 'controls')); - let count = 0; - const scratch = (id) => - privateDirectory(join(controls, `${String((count += 1)).padStart(2, '0')}-${id}`)); - const cli = join(candidate, 'apps', 'cli'); - const tarball = join(artifact, receipt.tarball); - const sealed = readPrivateManifest(staging); - const distDigests = () => inventory(join(cli, 'dist')); - - // The chunks the anchors' sources lie in, by their staged maps; the anchor sources' upstream map. - const anchorSources = new Set(ANCHORS.map((anchor) => anchor.source)); - const mapsOf = (name) => - JSON.parse(readOrdinaryFile(join(staging, 'maps', name), 16 * MIB).toString('utf8')); - const chunks = readdirSync(join(staging, 'maps')) - .filter((name) => mapsOf(name).sources.some((path) => anchorSources.has(path))) - .map((name) => name.slice(0, -'.map'.length)) - .sort(); - assert.ok(chunks.length > 0, 'PACKED_CLI_MAP_CONTROL_NO_ANCHOR_CHUNK'); - const plain = sealed.outputs - .map((output) => basename(output.path)) - .filter((name) => !chunks.includes(name)) - .sort(); - - /** A copy of the sealed staging re-recorded for the candidate as it now is, then symbolicated. */ - const resealed = async (id, { forceDirty = dirty, changeMap } = {}) => { - const root = scratch(id); - const copy = privateDirectory(join(root, 'staging')); - privateDirectory(join(copy, 'maps')); - const finalized = JSON.parse( - readOrdinaryFile(join(staging, 'finalized.json'), MIB).toString('utf8'), - ); - finalized.maps = finalized.maps.map((entry) => { - const name = basename(entry.path); - let bytes = readOrdinaryFile(join(staging, 'maps', name), 16 * MIB); - if (changeMap !== undefined && name === `${changeMap.chunk}.map`) { - bytes = Buffer.from(JSON.stringify(changeMap.change(JSON.parse(bytes.toString('utf8'))))); - } - writeFileSync(join(copy, 'maps', name), bytes, { mode: 0o600, flag: 'wx' }); - return { path: entry.path, bytes: bytes.length, sha256: sha256(bytes) }; - }); - finalized.outputs = finalized.outputs.map((entry) => { - const bytes = readOrdinaryFile(join(cli, ...entry.path.split('/')), 8 * MIB); - return { path: entry.path, bytes: bytes.length, sha256: sha256(bytes) }; - }); - writeFileSync(join(copy, 'finalized.json'), JSON.stringify(finalized), { - mode: 0o600, - flag: 'wx', - }); - const digests = new Map( - finalized.outputs.map((output) => [`package/${output.path}`, output.sha256]), - ); - const archive = { - sha256: receipt.tarballSha256, - package: receipt.package, - files: receipt.files.map((file) => - digests.has(file.path) ? { ...file, sha256: digests.get(file.path) } : file, - ), - }; - await sealPrivateManifest({ - candidate, - staging: copy, - checkout, - source, - dirty: forceDirty, - archive, - }); - const work = privateDirectory(join(root, 'work')); - return refusal(() => - symbolicateCandidate({ candidate, staging: copy, checkout, work, archive }), - ); - }; - - /** One pack of the candidate as it now is, into a fresh staging and artifact directory. */ - const pack = async (id, stagingOverride) => { - const root = scratch(id); - const destination = privateDirectory(join(root, 'artifact')); - const fresh = stagingOverride ?? privateDirectory(join(root, 'staging')); - const observed = await refusal(() => - pnpm(['pack', '--pack-destination', destination], { - cwd: cli, - env: { ...env, [CANDIDATE_STAGING_ENV]: fresh }, - }), - ); - return { observed, destination, staging: fresh, root }; - }; - - /** Seal a fresh pack's tarball with fresh work, as the producer would. */ - const seal = (packed) => - refusal(() => - sealPackage(packed.destination, { - repository: candidate, - staging: packed.staging, - source, - dirty, - checkout, - work: privateDirectory(join(packed.root, 'work')), - }), - ); - - // Post-receipt rebuild refusals that need the sealed state come first. - { - // A second prepack into the sealed staging refuses before any build, changing nothing. - const before = { dist: distDigests(), staging: inventory(staging, new Set(['controls'])) }; - const second = await pack('second-prepack', staging); - const unchanged = - JSON.stringify(distDigests()) === JSON.stringify(before.dist) && - JSON.stringify(inventory(staging, new Set(['controls']))) === - JSON.stringify(before.staging) && - readdirSync(second.destination).length === 0 && - second.observed.builds === 0; - record( - 'second-prepack', - 'STAGING_NOT_EMPTY', - unchanged && - second.observed.diagnostics.length === 1 && - second.observed.diagnostics[0] === '✗ CLI candidate finalization refused: STAGING_NOT_EMPTY' - ? 'STAGING_NOT_EMPTY' - : `MARKERS_${second.observed.code}`, - ); - } - { - // A re-seal into the artifact directory, or of the sealed staging beside a copied tarball, refuses - // before any capture runs. - const receiptDigest = sha256( - readOrdinaryFile(join(artifact, 'package-test-receipt.json'), MIB), - ); - const root = scratch('re-seal'); - const works = [privateDirectory(join(root, 'work-1')), privateDirectory(join(root, 'work-2'))]; - const again = await refusal(() => - sealPackage(artifact, { - repository: candidate, - staging, - source, - dirty, - checkout, - work: works[0], - }), - ); - const copied = privateDirectory(join(root, 'artifact')); - cpSync(tarball, join(copied, receipt.tarball)); - const beside = await refusal(() => - sealPackage(copied, { - repository: candidate, - staging, - source, - dirty, - checkout, - work: works[1], - }), - ); - const quiet = - works.every((work) => readdirSync(work).length === 0) && - sha256(readOrdinaryFile(join(artifact, 'package-test-receipt.json'), MIB)) === receiptDigest; - record( - 're-seal-receipt', - 'PACKED_CLI_ALREADY_SEALED', - quiet ? again.code : `MARKERS_${again.code}`, - ); - record( - 're-seal-staging', - 'PACKED_CLI_ALREADY_SEALED', - quiet ? beside.code : `MARKERS_${beside.code}`, - ); - } - - // The re-recording technique alone refuses nothing: the unchanged copy reaches every anchor. - record('re-recorded-baseline', 'ACCEPTED', (await resealed('re-recorded-baseline')).code); - - // Wrong JS by line and by column, a wrong map and permuted sources, on every anchor-bearing chunk. - for (const [index, chunk] of chunks.entries()) { - const path = join(cli, 'dist', chunk); - for (const [kind, change] of [ - ['js-line', shiftLines], - ['js-column', shiftColumns], - ]) { - const saved = snapshot(path); - try { - writeFileSync(path, change(saved.bytes.toString('utf8'))); - record( - `${kind}-${index + 1}`, - 'PACKED_CLI_SYMBOLICATION_ANCHOR_UNMAPPED', - (await resealed(kind)).code, - ); - } finally { - restore(saved); - } - } - record( - `map-line-${index + 1}`, - 'PACKED_CLI_SYMBOLICATION_ANCHOR_UNMAPPED', - ( - await resealed('map-line', { - changeMap: { chunk, change: (map) => ({ ...map, mappings: `;${map.mappings}` }) }, - }) - ).code, - ); - record( - `sources-rotated-${index + 1}`, - 'PACKED_CLI_SYMBOLICATION_ANCHOR_UNMAPPED', - ( - await resealed('sources-rotated', { - changeMap: { chunk, change: (map) => ({ ...map, sources: rotateSources(map.sources) }) }, - }) - ).code, - ); - } - - // A wrong source revision: clean, the Git object refuses the copy; dirty, the anchor positions move. - { - const saved = snapshot(join(candidate, ...ANCHORS[1].source.split('/'))); - try { - writeFileSync(saved.path, `// shifted\n${saved.bytes.toString('utf8')}`); - record( - 'source-clean', - 'PACKED_CLI_SYMBOLICATION_SOURCE_REVISION', - (await resealed('source-clean', { forceDirty: false })).code, - ); - record( - 'source-dirty', - 'PACKED_CLI_SYMBOLICATION_ANCHOR_UNMAPPED', - (await resealed('source-dirty', { forceDirty: true })).code, - ); - } finally { - restore(saved); - } - } - - // A rebuilt dist that differs from the manifest: the sealed state verifies; a changed chunk no frame - // lands in, and a real rebuild from a changed source, refuse. - record( - 'sealed-dist', - 'ACCEPTED', - (await refusal(() => verifySealedDist({ candidate, manifest: sealed, tarball }))).code, - ); - if (plain.length > 0) { - const saved = snapshot(join(cli, 'dist', plain[0])); - try { - writeFileSync(saved.path, Buffer.concat([saved.bytes, Buffer.from(' ')])); - record( - 'dist-plain-chunk', - 'PACKED_CLI_MANIFEST_DIST_DRIFT', - (await refusal(() => verifySealedDist({ candidate, manifest: sealed, tarball }))).code, - ); - } finally { - restore(saved); - } - } - const entry = snapshot(join(cli, 'src', 'index.ts')); - try { - writeFileSync( - entry.path, - `${entry.bytes.toString('utf8')}\nglobalThis.relaviumMapControls = 1;\n`, - ); - const rebuilt = await pack('rebuild'); - const changed = sealed.outputs.some((output) => { - try { - return ( - sha256(readOrdinaryFile(join(cli, ...output.path.split('/')), 8 * MIB)) !== output.sha256 - ); - } catch { - return true; - } - }); - const verified = await refusal(() => - verifySealedDist({ candidate, manifest: sealed, tarball }), - ); - record( - 'dist-rebuilt', - 'PACKED_CLI_MANIFEST_DIST_DRIFT', - rebuilt.observed.code === 'ACCEPTED' && changed - ? verified.code - : `MARKERS_${rebuilt.observed.code}`, - ); - } finally { - restore(entry); - } - - // Upstream maps: a missing, a malformed and a value-truncated map refuse in the finalizer after the - // bundler silently succeeded; a byte-truncated map fails the build; a shifted but valid map is - // accepted there and refused by symbolication. - const upstream = ANCHORS[0].source - .replace(/^packages\/core\/src\//u, 'packages/core/dist/') - .replace(/\.ts$/u, '.js'); - const upstreamMap = join(candidate, ...`${upstream}.map`.split('/')); - const saved = snapshot(upstreamMap); - const original = JSON.parse(saved.bytes.toString('utf8')); - const refusedLine = (code, subject) => - `✗ CLI candidate finalization refused: ${code} (${subject})`; - for (const [id, corrupt, expected] of [ - ['upstream-missing', () => rmSync(upstreamMap), refusedLine('UPSTREAM_MAP_MISSING', upstream)], - [ - 'upstream-shape', - () => writeFileSync(upstreamMap, JSON.stringify({ ...original, sourcesContent: [null] })), - refusedLine('UPSTREAM_MAP_SHAPE', `${upstream}.map`), - ], - [ - 'upstream-index', - () => - writeFileSync( - upstreamMap, - JSON.stringify({ ...original, mappings: `A${vlq(original.sources.length)}AA` }), - ), - refusedLine('UPSTREAM_MAP_MAPPINGS', `${upstream}.map`), - ], - [ - // A map that is no longer JSON fails the bundler's own build, before the finalizer reads it. - 'upstream-cut-bytes', - () => writeFileSync(upstreamMap, saved.bytes.subarray(0, Math.floor(saved.bytes.length / 2))), - '✗ CLI build failed', - ], - [ - 'upstream-cut-value', - () => - writeFileSync( - upstreamMap, - JSON.stringify({ ...original, mappings: cutInsideValue(original.mappings) }), - ), - refusedLine('UPSTREAM_MAP_MAPPINGS', `${upstream}.map`), - ], - ]) { - try { - corrupt(); - const packed = await pack(id); - // The bundler silently succeeded once, nothing was finalized, staged or packed, and the one - // prepack diagnostic is the expected fixed line. - const markers = [ - packed.observed.builds, - packed.observed.finalized, - readdirSync(packed.staging).length, - readdirSync(packed.destination).length, - packed.observed.diagnostics.length, - ]; - const line = packed.observed.diagnostics.at(-1); - record( - id, - expected, - markers.join() === '1,0,0,0,1' && line === expected - ? expected - : `MARKERS(${markers.join()}) ${line ?? packed.observed.code}`, - ); - } finally { - restore(saved); - } - } - try { - const mappings = shiftOriginalColumns(original.mappings, { - sources: original.sources.length, - names: original.names.length, - }); - // The shift is strictly valid: the finalizer's own checks admit it. - const lines = decodeMappings(mappings, { - sources: original.sources.length, - names: original.names.length, - }); - assertGeneratedRange( - lines, - lineLengths( - readOrdinaryFile(join(candidate, ...upstream.split('/')), 16 * MIB).toString('utf8'), - ), - ); - assertOriginalRange( - lines, - original.sources.map((path) => - lineLengths( - readOrdinaryFile(join(dirname(upstreamMap), ...path.split('/')), 16 * MIB).toString( - 'utf8', - ), - ), - ), - ); - writeFileSync(upstreamMap, JSON.stringify({ ...original, mappings })); - const packed = await pack('upstream-shifted'); - const accepted = packed.observed.code === 'ACCEPTED'; - record( - 'upstream-shifted', - 'PACKED_CLI_SYMBOLICATION_ANCHOR_UNMAPPED', - accepted ? (await seal(packed)).code : `MARKERS_${packed.observed.code}`, - ); - } finally { - restore(saved); - } - - // Restoration: every corrupted file is back, so a fresh pack seals again. - const restored = await pack('restoration'); - record( - 'restoration', - 'ACCEPTED', - restored.observed.code === 'ACCEPTED' - ? (await seal(restored)).code - : `MARKERS_${restored.observed.code}`, - ); - return results; -} - -/** - * Run the controls through one real production of `repository` and return their results with the - * source identity. The developer checkout must be the same before and after. - * @param {{repository: string, environment: Record, - * temporary?: string}} options - */ -export async function runMapControls({ repository, environment, temporary = tmpdir() }) { - const before = await sourceIdentity(repository); - const root = realpathSync.native(mkdtempSync(join(temporary, 'relavium-map-controls-'))); - chmodSync(root, 0o700); - let results = []; - let emitted; - try { - const produced = await producePackage({ - repository, - destination: join(root, 'artifact'), - environment, - temporary, - ports: { - emit: (options) => { - emitted = { env: options.env, pnpm: options.pnpm }; - return emitFreshPackage(options); - }, - seal: async (artifact, inputs) => { - const receipt = await sealPackage(artifact, inputs); - results = await runControls({ - candidate: inputs.repository, - staging: inputs.staging, - checkout: inputs.checkout, - source: inputs.source, - dirty: inputs.dirty, - receipt, - artifact, - ...emitted, - }); - return receipt; - }, - }, - }); - const after = await sourceIdentity(repository); - assert.deepEqual( - [after.source, after.status, after.unchecked], - [before.source, before.status, before.unchecked], - 'PACKED_CLI_MAP_CONTROL_CHECKOUT_CHANGED', - ); - return { source: produced.receipt.source, dirty: produced.identity.dirty, results }; - } finally { - rmSync(root, { recursive: true, force: true }); - } -} - -if ( - process.argv[1] !== undefined && - realpathSync.native(process.argv[1]) === fileURLToPath(import.meta.url) -) { - const started = Date.now(); - let outcome; - try { - outcome = await runMapControls({ - repository: fileURLToPath(new URL('../../', import.meta.url)), - environment: process.env, - }); - } catch (error) { - for (const line of productionFailure(error)) console.error(line); - process.exit(1); - } - const failed = outcome.results.filter((result) => result.observed !== result.expected); - console.log( - JSON.stringify({ - scope: 'product copies of one real package-test production', - source: outcome.source, - dirty: outcome.dirty, - controls: outcome.results.length, - failed: failed.map((result) => result.id), - results: outcome.results, - seconds: Math.round((Date.now() - started) / 1000), - }), - ); - if (failed.length > 0) { - console.error( - `✗ CLI product map controls failed: ${failed.map((result) => result.id).join(', ')}`, - ); - process.exit(1); - } -} +const started = Date.now(); +let outcome; +try { + outcome = await runMapControls({ + repository: fileURLToPath(new URL('../../', import.meta.url)), + environment: process.env, + }); +} catch (error) { + for (const line of productionFailure(error)) console.error(line); + process.exit(1); +} +process.exitCode = report(outcome, { seconds: Math.round((Date.now() - started) / 1000) }); diff --git a/tools/cli-package/map-controls.test.mjs b/tools/cli-package/map-controls.test.mjs index 1666fb1e..dd9fb4d5 100644 --- a/tools/cli-package/map-controls.test.mjs +++ b/tools/cli-package/map-controls.test.mjs @@ -1,13 +1,16 @@ import assert from 'node:assert/strict'; import { test } from 'node:test'; import { + FIXED_CONTROLS, cutInsideValue, encodeMappings, + report, rotateSources, shiftColumns, shiftLines, shiftOriginalColumns, -} from './map-controls.mjs'; + verdict, +} from './map-controls-core.mjs'; import { SourceMapError, decodeMappings } from './source-map.mjs'; // Importing the runner runs no control: only its command-line entry point does. @@ -128,3 +131,73 @@ test('a value cut after its first continuation digit never ends', () => { refuses('PACKED_CLI_MAP_CONTROL_NO_CONTINUATION'), ); }); + +/** A passing run's results: every fixed control and four for each of `chunks` anchor-bearing chunks. */ +function passing(chunks) { + const ids = [ + ...FIXED_CONTROLS, + ...Array.from({ length: chunks }, (_, index) => + ['js-line', 'js-column', 'map-line', 'sources-rotated'].map((kind) => `${kind}-${index + 1}`), + ).flat(), + ]; + return ids.map((id) => ({ id, expected: 'CODE', observed: 'CODE' })); +} + +test('the verdict passes only a complete run whose every control had its expected outcome', () => { + assert.equal(verdict({ results: passing(2), chunks: 2 }).passed, true); + assert.equal(verdict({ results: passing(2), chunks: 2 }).expected, FIXED_CONTROLS.length + 8); + // No anchor-bearing chunk. + assert.equal(verdict({ results: passing(0), chunks: 0 }).passed, false); + // A control that did not run, here a fixed one and a per-chunk one. + for (const id of ['dist-plain-chunk', 'sources-rotated-2']) { + const judged = verdict({ results: passing(2).filter((result) => result.id !== id), chunks: 2 }); + assert.deepEqual([judged.passed, judged.missing], [false, [id]]); + } + // A control that ran twice, or one that is not expected. + const twice = verdict({ + results: [...passing(1), { id: 'restoration', expected: 'A', observed: 'A' }], + chunks: 1, + }); + assert.deepEqual([twice.passed, twice.unexpected], [false, ['restoration']]); + const stray = verdict({ + results: [...passing(1), { id: 'js-line-2', expected: 'A', observed: 'A' }], + chunks: 1, + }); + assert.deepEqual([stray.passed, stray.unexpected], [false, ['js-line-2']]); + // An observed outcome other than the expected one. + const results = passing(1); + results[3] = { ...results[3], observed: 'OTHER' }; + const failed = verdict({ results, chunks: 1 }); + assert.deepEqual([failed.passed, failed.failed], [false, [results[3].id]]); +}); + +test('the report prints one summary and returns 0 only for a passing run', () => { + const run = (outcome) => { + const written = []; + const warned = []; + const code = report(outcome, { + seconds: 3, + write: (line) => written.push(line), + warn: (line) => warned.push(line), + }); + return { code, written, warned }; + }; + const passed = run({ source: 'a'.repeat(40), dirty: false, results: passing(1), chunks: 1 }); + assert.deepEqual([passed.code, passed.warned], [0, []]); + assert.equal(passed.written.length, 1); + const summary = JSON.parse(passed.written[0]); + assert.deepEqual( + [summary.failed, summary.missing, summary.unexpected, summary.chunks], + [[], [], [], 1], + ); + const results = passing(1).filter((result) => result.id !== 'seal-verify'); + results[0] = { ...results[0], observed: 'OTHER' }; + const failed = run({ source: 'a'.repeat(40), dirty: false, results, chunks: 1 }); + assert.equal(failed.code, 1); + assert.deepEqual(failed.warned, [ + `✗ CLI product map controls failed: ${results[0].id}, seal-verify (missing)`, + ]); + const empty = run({ source: 'a'.repeat(40), dirty: false, results: [], chunks: 0 }); + assert.equal(empty.code, 1); + assert.equal(empty.warned.length, 1); +}); diff --git a/tools/cli-package/seal.mjs b/tools/cli-package/seal.mjs index ee6b8730..9fcc2262 100644 --- a/tools/cli-package/seal.mjs +++ b/tools/cli-package/seal.mjs @@ -107,8 +107,9 @@ function present(path) { /** * Inspect the tarball, then seal the candidate's private map manifest, then symbolicate actual packed * frames against it, then verify that `dist` and the tarball are still the manifest's, and only then - * write the receipt: a refusal at any step leaves no receipt. A candidate is sealed once: a receipt - * beside the tarball or a manifest in the staging refuses before anything is read or run. + * write the receipt: a refusal at any step leaves no receipt. An artifact directory that already holds + * a receipt, or a staging that already holds a manifest, is never sealed again: either refuses before + * the inspection and any capture. * @param {string} artifactDirectory the public artifact directory holding exactly one tarball * @param {{repository: string, staging: string, source: string, dirty: boolean, checkout: string, * work: string, ports?: {manifest?: typeof sealPrivateManifest, diff --git a/tools/cli-package/seal.test.mjs b/tools/cli-package/seal.test.mjs index 026141c9..ecd47622 100644 --- a/tools/cli-package/seal.test.mjs +++ b/tools/cli-package/seal.test.mjs @@ -633,6 +633,120 @@ test('a sealed candidate is never sealed again, and nothing is read or run first }); }); +/** The manifest a real seal would hold for the fixture's bytes: its sorted outputs and the tarball. */ +function sealedManifest(root, artifact) { + const digest = (bytes) => createHash('sha256').update(bytes).digest('hex'); + const dist = join(root, 'apps/cli/dist'); + return { + outputs: readdirSync(dist) + .sort() + .map((name) => ({ path: `dist/${name}`, sha256: digest(readFileSync(join(dist, name))) })), + tarballSha256: digest(readFileSync(join(artifact, 'relavium-0.1.1.tgz'))), + }; +} +const sealInputs = (root, staging) => ({ + repository: root, + staging, + source: SOURCE, + dirty: false, + checkout: root, + work: join(dirname(staging), 'work'), +}); + +test('without a verify port the seal checks dist and the tarball before the receipt', async () => { + // Unchanged, they verify and the receipt is written. + await withCandidate(async ({ root, staging, artifact }) => { + const manifest = sealedManifest(root, artifact); + await withOrigin({}, () => + sealPackage(artifact, { + ...sealInputs(root, staging), + ports: { manifest: async () => manifest, symbolicate: async () => {} }, + }), + ); + assert.equal(existsSync(receiptPath(artifact)), true); + }); + // A chunk or the tarball changed during the capture refuses, and no receipt is written. + for (const [change, code] of [ + [ + (root) => writeFileSync(join(root, 'apps/cli/dist/chunk-ABC.js'), 'export {}; \n'), + 'DIST_DRIFT', + ], + [ + (_root, artifact) => writeFileSync(join(artifact, 'relavium-0.1.1.tgz'), 'other'), + 'TARBALL_CHANGED', + ], + ]) { + await withCandidate(async ({ root, staging, artifact }) => { + const manifest = sealedManifest(root, artifact); + await withOrigin({}, () => + assert.rejects( + sealPackage(artifact, { + ...sealInputs(root, staging), + ports: { + manifest: async () => manifest, + symbolicate: async () => change(root, artifact), + }, + }), + (error) => error instanceof ManifestError && error.code === code, + ), + ); + assert.equal(existsSync(receiptPath(artifact)), false); + }); + } +}); + +test('the already-sealed refusal precedes the inspection, and other extras keep their own code', async () => { + // A staged manifest beside an archive the inspection would refuse: the seal refusal comes first. + await withCandidate(async ({ root, staging, artifact, files }) => { + writeFileSync(join(staging, 'manifest.json'), '{}'); + files.set('package/LICENSE', Buffer.from('changed\n')); + writeFileSync(join(artifact, 'relavium-0.1.1.tgz'), gzipSync(ustar(files))); + await withOrigin({}, () => + assert.rejects(sealPackage(artifact, sealInputs(root, staging)), { + message: 'PACKED_CLI_ALREADY_SEALED', + }), + ); + }); + // A receipt beside the tarball and no candidate at all: still the seal refusal. + await withCandidate(async ({ staging, artifact }) => { + writeFileSync(receiptPath(artifact), '{}'); + await withOrigin({}, () => + assert.rejects(sealPackage(artifact, sealInputs(join(artifact, 'no-candidate'), staging)), { + message: 'PACKED_CLI_ALREADY_SEALED', + }), + ); + }); + // Any other extra file is the public artifact check's refusal, never ALREADY_SEALED. + await withCandidate(async ({ root, staging, artifact }) => { + writeFileSync(join(artifact, 'notes.txt'), 'extra'); + await withOrigin({}, () => + assert.rejects(sealPackage(artifact, sealInputs(root, staging)), { + message: 'PACKED_CLI_UNEXPECTED_PUBLIC_ARTIFACT', + }), + ); + }); +}); + +test('a receipt write failing other than EEXIST keeps its own error and leaves no receipt', async () => { + await withCandidate(async ({ root, staging, artifact }) => { + await withOrigin({}, () => + assert.rejects( + sealPackage(artifact, { + ...sealInputs(root, staging), + ports: { + manifest: async () => SEALED, + symbolicate: async () => {}, + // The artifact directory disappears before the receipt is written. + verify: async () => rmSync(artifact, { recursive: true, force: true }), + }, + }), + (error) => error?.code === 'ENOENT', + ), + ); + assert.equal(existsSync(receiptPath(artifact)), false); + }); +}); + test('both steps receive the candidate, the original checkout and the dirty flag as given', async () => { await withCandidate(async ({ root, staging, artifact }) => { const calls = []; From 0f73c2532176f0fdbd3f2549203e51691c90898b Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sat, 24 Oct 2026 23:02:00 +0300 Subject: [PATCH 51/97] fix(repo): address the first Step 9a.2b review round Round 1 at 3e6ae33c was NONCLEAN. - Medium, a regression in the author's own record: 9f8b0890 replaced the start of deferred-tasks.md's "Phase 2.6.5 deferrals" heading instead of inserting before it, so the approved CR-70, CR-93 and CR-95 deferrals fell under the native-identity section. The heading is restored exactly; no other line any 9a.2b commit removed was lost. - Medium: nothing tested the entry point's exit code. A test now runs the real map-controls.mjs beside a stand-in core: exit 0 for a passing run, exit 1 naming a failed or missing control, and exit 1 with only the fixed code for a run that throws. - The runner no longer aborts without a summary when a refusal breaks: an aborted entry and every missing control are reported and the broken control is named. Every refused control build that ran the bundler has its dist removed. The seventeen fixed controls are pinned. - Docs: the contract's earlier backing sentence now says when an artifact is backed; "restores the file" excepts dist; the register's counts (25 controls, 932 tests, four lows and one medium, fourteen required jobs) are corrected; the status lines name the NI increment. Gates: lint:tools, typecheck:tools, prettier, test:acceptance-tools (934 of 934 under CI=true GITHUB_ACTIONS=true), the real runner (25 of 25) and smoke:cli-package; the entry mutants and a broken STAGING_NOT_EMPTY are each caught. Refs: ADR-0111, #46 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 +- CLAUDE.md | 4 +- docs/reference/cli/packaging.md | 9 +- docs/roadmap/current.md | 6 +- docs/roadmap/deferred-tasks.md | 3 +- .../phase-2.5.5-hardening-and-remediation.md | 35 ++++++-- tools/cli-package/map-controls-core.mjs | 69 ++++++++++----- tools/cli-package/map-controls.test.mjs | 86 +++++++++++++++++++ 8 files changed, 175 insertions(+), 41 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 03b09cb2..e039b805 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,7 +117,7 @@ explicit; source CI does not certify later documentation commits. The joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now -accepted on `development`, below, as is Step 9a.2a; Step 9a.2b is implemented and under review; Steps 9b–19 have not started) and whole-wave acceptance remains Step 19. +accepted on `development`, below, as is Step 9a.2a; Step 9a.2b is implemented and under review; the native-identity increment NI and Steps 9b–19 have not started) and whole-wave acceptance remains Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification @@ -130,7 +130,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-implementation-checkpoint--2026-10-24)). `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-implementation-checkpoint--2026-10-24)). The native-identity increment NI (a Windows file-identity security fix the maintainer approved on 2026-10-24, register row NI) follows it, before Step 9b. `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. See [docs/roadmap/current.md](docs/roadmap/current.md) for live status. diff --git a/CLAUDE.md b/CLAUDE.md index f648f233..0b3482d4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -206,7 +206,7 @@ explicit; source CI does not certify later documentation commits. The joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now -accepted on `development`, below, as is Step 9a.2a; Step 9a.2b is implemented and under review; Steps 9b–19 have not started) and whole-wave acceptance remains Step 19. +accepted on `development`, below, as is Step 9a.2a; Step 9a.2b is implemented and under review; the native-identity increment NI and Steps 9b–19 have not started) and whole-wave acceptance remains Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification @@ -219,7 +219,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-implementation-checkpoint--2026-10-24)). `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-implementation-checkpoint--2026-10-24)). The native-identity increment NI (a Windows file-identity security fix the maintainer approved on 2026-10-24, register row NI) follows it, before Step 9b. `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. For live status, per-PR history, milestone dates, and open obligations, see the canonical diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index d2e782d4..af49b4dd 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -160,8 +160,10 @@ bounded retry evidence. These runners are implemented; required platform executi and independent acceptance are recorded separately in the Wave 3 register. Since Step 9a.2b the receipt records `package_test`: before writing it the producer verified its -maps and symbolicated actual packed frames (Steps 9a.1 and 9a.2a), and the required `ci` at the -same commit runs the product map controls (below). Release custody remains Steps 9b–9d. Native fatal-owner integration is covered by [Step 8 acceptance](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17). +maps and symbolicated actual packed frames (Steps 9a.1 and 9a.2a), and the `ci` workflow requires the +product map controls; an artifact is backed by them only where a successful `ci` run exists at exactly +its commit (below). Release custody remains Steps 9b–9d. Native fatal-owner integration is covered by +[Step 8 acceptance](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17). No intermediate local result closes Step 9 or certifies a release. ### Candidate finalization — Step 9 @@ -436,7 +438,8 @@ package-test producer, adds the product map controls and only then flips the rec anything is built (`PACKED_CLI_MAP_CONTROL_ORIGIN`). Inside the seal, after the baseline receipt and while the private staging still exists, each control corrupts one product copy — the owned candidate tree, or a copy of its private staging re-recorded so that only the layer under test can refuse — - requires that control's exact refusal and restores the file by digest: + requires that control's exact refusal and restores every file it corrupted by digest (`dist` itself + is replaced by the next build): - a second prepack into the sealed staging; a re-seal of the artifact directory against a copy of the staging without its manifest, and of the sealed staging beside a tarball copied without its receipt, each isolating one half of `PACKED_CLI_ALREADY_SEALED` with no capture run; diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index 24fac7d8..96dc5138 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -138,7 +138,7 @@ The [deferred register](deferred-tasks.md) retains its existing limits; closure **Wave 3 Steps 1–8 merged to `main` on 2026-10-18 as [PR #95](https://github.com/HodeTech/Relavium/pull/95), merge commit `6d2a7372` (22:44 UTC). Wave 3 itself is not complete: Steps 9–19 had not started at -that merge (Steps 9a.1 and 9a.2a are now accepted on `development` and Step 9a.2b is implemented and under review; Steps 9b–19 have not started) and whole-wave acceptance +that merge (Steps 9a.1 and 9a.2a are now accepted on `development` and Step 9a.2b is implemented and under review; the native-identity increment NI and Steps 9b–19 have not started) and whole-wave acceptance remains Step 19.** The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) joins the completed scope, every review attribution, the independent review's dispositions and the publication state observed at merge. It was the maintainer-authorized, scoped publication checkpoint. @@ -172,7 +172,7 @@ bar and round cap, as its medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, by the author alone). Its other findings are recorded in -[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is independently accepted on `development` at `cf97b7f1` (2026-10-24): its round 13 reported no blocker, high or medium finding and its security sitting was clean, and the round's low findings are recorded in deferred-tasks.md ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)); Step 9a.2 is split into 9a.2a and 9a.2b, and Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24) after three rounds, the third with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)); Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is implemented on `development` and under review ([checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-implementation-checkpoint--2026-10-24)); the substrate's +[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is independently accepted on `development` at `cf97b7f1` (2026-10-24): its round 13 reported no blocker, high or medium finding and its security sitting was clean, and the round's low findings are recorded in deferred-tasks.md ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)); Step 9a.2 is split into 9a.2a and 9a.2b, and Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24) after three rounds, the third with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)); Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is implemented on `development` and under review ([checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-implementation-checkpoint--2026-10-24)), and the native-identity increment NI (a Windows file-identity security fix the maintainer approved on 2026-10-24, register row NI) follows it, before Step 9b; the substrate's 9a.1–9d increments carry a dedicated archive/crypto security sitting. Steps 10–19 follow the [increment table](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) and end with whole-wave acceptance at Step 19. `development` is pushed after each accepted increment; PR #95 no longer bounds the @@ -1141,7 +1141,7 @@ an ESLint selector rather than a list. ### Wave 3 — Clear the ground -**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Steps 9a.1 and 9a.2a accepted on `development` (2026-10-24), Step 9a.2b under review, Steps 9b–19 NOT STARTED — 2026-10-24.** +**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Steps 9a.1 and 9a.2a accepted on `development` (2026-10-24), Step 9a.2b under review, NI and Steps 9b–19 NOT STARTED — 2026-10-24.** ADR-0110–0113, their linked contracts and the nineteen-increment plan were approved on 2026-10-16. [Steps 1–8 were independently accepted](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) on `development`, with whole Step 8 reviewed at `c83c8575`, and published as the scoped checkpoint diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index 6fec109e..67759089 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -772,7 +772,8 @@ increment, after 9a.2b and before 9b (register row NI), rather than a deferral; `nativeFile`). **Re-check:** every identity is read with `{ bigint: true }` and compared as `bigint` (or its exact string), with a replacement test per site that fails when the comparison goes back to a number; the Windows job passes the fs-policy test repeatedly. *(high · the five files above)* -, recorded 2026-09-06; `CR-70`, `CR-94` (remainder) and the in-turn overflow continuation, recorded 2026-09-14; the errored-or-aborted turn residual, the pre-upgrade free-page bytes and the estimator's calibration, recorded 2026-09-18 + +## Phase 2.6.5 deferrals — `CR-93` and `CR-95` (long half), recorded 2026-09-06; `CR-70`, `CR-94` (remainder) and the in-turn overflow continuation, recorded 2026-09-14; the errored-or-aborted turn residual, the pre-upgrade free-page bytes and the estimator's calibration, recorded 2026-09-18 **Phase closure — 2026-10-11:** W8 merged to `main` in PR #92 and Phase 2.6.5 is complete with approved deferrals, 49/51 items closed. The diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 5717eab5..1401daa5 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -9773,7 +9773,7 @@ second) matched this checkpoint. The round's three verified findings (one low, t corrected in the acceptance commit, which changes no code, and the two test gaps stay open there. The receipt stays `compiled_package_test`; Step 9a.2b is next. -**Push CI.** `development` was pushed at `f65999a6` (the acceptance commit on top of `a92a7602`). All sixteen +**Push CI.** `development` was pushed at `f65999a6` (the acceptance commit on top of `a92a7602`). All fourteen required jobs of [run 40105286543](https://github.com/HodeTech/Relavium/actions/runs/40105286543) passed, among them the originating pack, the four same-tarball installed-CLI consumers and their aggregate, lint, typecheck and test, the engine coverage floor and the causal-control families. The advisory Windows job failed on a security assertion @@ -9823,9 +9823,10 @@ step after `smoke:cli-package` and the workflow pins; and only then the `package `PACKAGE_TEST_MODE` shared by the seal and both receipt validators, `compiled_package_test` refused), with the [packaging contract](../../reference/cli/packaging.md#product-map-controls-and-the-package-test-receipt--step-9a2b), the tooling README and the status lines. No runtime dependency or ADR is added; `release.yml` gets no map -controls job, as ruled. On this host the runner's 24 controls pass in about 30 seconds on one real production, -`smoke:cli-package` passes with the flipped receipt through the real producer and consumers, and -`test:acceptance-tools` passes 911 of 911. +controls job, as ruled. At the implementation head (`77ee5d0a`) the runner's 24 controls passed in about 30 seconds +on one real production, `smoke:cli-package` passed with the flipped receipt through the real producer and +consumers, and `test:acceptance-tools` passed 911 of 911; after the rigor pass the runner records 25 (the +seventeen fixed controls plus four for each of the two anchor-bearing chunks) and the suite runs 932 tests. **Internal rigor pass, 2026-10-24.** Under the rigor-before-review rule a pass covered the whole increment at `77ee5d0a` before its first round: three mutation sweeps (the seal refusals and the flip; the runner, including @@ -9848,10 +9849,28 @@ for every dispatch — corrected in the contract and the workflow comments, with `ci` run recorded in deferred-tasks.md. Lows corrected include an early refusal of a half-configured CI environment, removing a refused control build's `dist`, tests of the already-sealed refusal's order, of a failed receipt write, of the version's start anchor and of `verifySealedDist`'s every output and read bounds, and the -summary, checkout and already-sealed wording. Five lows are recorded in +summary, checkout and already-sealed wording. Four lows, and one medium for Step 9d, are recorded in [deferred-tasks.md](../deferred-tasks.md#wave-3-step-9a2b--recorded-low-findings-2026-10-24). The five mutants the corrections target now fail the real runner, each on its own control or self-check. -**Remaining gates.** The full `pnpm run ci`, then one NEW complete cumulative round with no blocker, high or medium -finding and no regression, together with the separate scoped security sitting at the same head and bar, and the -`map-controls` job green on the push CI. +**Author evidence at `3e6ae33c`.** The full `pnpm run ci` (exit 0, about 73 minutes, with `check:map-controls` +recording 25 of 25 expected controls on a clean tree, the real producer and consumer smoke and every causal-control +family). + +**Round 1 at `3e6ae33c` — NONCLEAN.** Five lenses and a critic, every finding adversarially verified. Four lenses +reported a regression in the author's own record (medium): commit `9f8b0890`, which recorded the native-identity +increment, replaced the start of deferred-tasks.md's "Phase 2.6.5 deferrals" heading instead of inserting before it, +so the approved CR-70, CR-93 and CR-95 deferrals fell under the new section; the heading is restored exactly, and an +audit of every line any 9a.2b commit removed found no other loss. The tests lens reported that nothing tested the +entry point's exit code (medium): a test now runs the real `map-controls.mjs` beside a stand-in core and requires +exit 0 for a passing run, exit 1 naming the control for a failed or missing one, and exit 1 with only the fixed code +for a run that throws. Lows corrected: the contract's earlier sentence that the same commit's `ci` runs the controls +now says when an artifact is backed; the runner no longer aborts without a summary when a refusal breaks (an +`aborted` entry and every missing control are reported, and the broken control is named); every refused control +build that ran the bundler has its `dist` removed; the seventeen fixed controls are pinned in a test; the "restores +the file" wording excepts `dist`; the checkpoint's counts (25 controls, 932 tests, four lows and one medium) and the +9a.2a push-CI job count (fourteen required jobs) are corrected; and the status lines name the NI increment. + +**Remaining gates.** Round 2 — one NEW complete cumulative round with no blocker, high or medium finding and no +regression, with the separate scoped security sitting at the same head and bar — then the full `pnpm run ci` and +the `map-controls` job green on the push CI. diff --git a/tools/cli-package/map-controls-core.mjs b/tools/cli-package/map-controls-core.mjs index 5881c74f..6de4a20d 100644 --- a/tools/cli-package/map-controls-core.mjs +++ b/tools/cli-package/map-controls-core.mjs @@ -4,7 +4,8 @@ * production runs at the checked-out commit. Inside its seal, after the baseline receipt and while the * private staging still exists, every named negative control corrupts one product copy — the * producer's own owned candidate tree, or a re-recorded copy of its private staging — and requires that - * control's exact refusal; each corrupted file is restored and checked by digest, and a restoration + * control's exact refusal; each corrupted file is restored and checked by digest (dist itself is + * replaced by the next build), and a restoration * production must then seal again. Staging copies stay under the staging, which the producer removes * first on success, failure and interrupt; a refused control build's `dist`, which holds the bundler's * maps, is removed at once. The developer's checkout is read, never written (Git may refresh its own @@ -207,14 +208,14 @@ function inventory(directory, skip = new Set()) { } /** - * Run every control inside the baseline seal. `context` holds the seal's own inputs, its receipt, the - * baseline artifact directory and the emitter's isolated environment and pnpm. - * @returns {Promise<{id: string, expected: string, observed: string}[]>} + * Run every control inside the baseline seal, recording into `state` as it goes, so a run that stops + * early still reports what ran. `context` holds the seal's own inputs, its receipt, the baseline + * artifact directory and the emitter's isolated environment and pnpm. + * @param {{results: {id: string, expected: string, observed: string}[], chunks: number}} state */ -async function runControls(context) { +async function runControls(context, state) { const { candidate, staging, checkout, source, dirty, receipt, artifact, env, pnpm } = context; - const results = []; - const record = (id, expected, observed) => results.push({ id, expected, observed }); + const record = (id, expected, observed) => state.results.push({ id, expected, observed }); const controls = privateDirectory(join(staging, 'controls')); let count = 0; const scratch = (id) => @@ -222,7 +223,14 @@ async function runControls(context) { const cli = join(candidate, 'apps', 'cli'); const tarball = join(artifact, receipt.tarball); const sealed = readPrivateManifest(staging); - const distDigests = () => inventory(join(cli, 'dist')); + // The dist's digests, or null once a broken refusal let a build replace or remove it. + const distDigests = () => { + try { + return inventory(join(cli, 'dist')); + } catch { + return null; + } + }; // The chunks the anchors' sources lie in, by their staged maps; the anchor sources' upstream map. const anchorSources = new Set(ANCHORS.map((anchor) => anchor.source)); @@ -233,6 +241,7 @@ async function runControls(context) { .map((name) => name.slice(0, -'.map'.length)) .sort(); assert.ok(chunks.length > 0, 'PACKED_CLI_MAP_CONTROL_NO_ANCHOR_CHUNK'); + state.chunks = chunks.length; const plain = sealed.outputs .map((output) => basename(output.path)) .filter((name) => !chunks.includes(name)) @@ -317,6 +326,11 @@ async function runControls(context) { env: { ...env, [CANDIDATE_STAGING_ENV]: fresh }, }), ); + // A refused build left the bundler's own maps in dist: none outlives the control. A refusal before + // any build (a second prepack into a sealed staging) leaves the sealed dist as it is. + if (observed.code !== 'ACCEPTED' && observed.builds > 0) { + rmSync(join(cli, 'dist'), { recursive: true, force: true }); + } return { observed, destination, staging: fresh, root }; }; @@ -598,8 +612,6 @@ async function runControls(context) { ); } finally { restore(saved); - // The refused build left the bundler's own maps in dist: none outlives the control. - rmSync(join(cli, 'dist'), { recursive: true, force: true }); } } try { @@ -681,7 +693,6 @@ async function runControls(context) { ? (await seal(restored)).code : `MARKERS_${restored.observed.code}`, ); - return { results, chunks: chunks.length }; } /** @@ -700,7 +711,7 @@ export async function runMapControls({ repository, environment, temporary = tmpd const before = await sourceIdentity(repository); const root = realpathSync.native(mkdtempSync(join(temporary, 'relavium-map-controls-'))); chmodSync(root, 0o700); - let ran = { results: [], chunks: 0 }; + const ran = { results: [], chunks: 0 }; let emitted; try { const produced = await producePackage({ @@ -715,16 +726,30 @@ export async function runMapControls({ repository, environment, temporary = tmpd }, seal: async (artifact, inputs) => { const receipt = await sealPackage(artifact, inputs); - ran = await runControls({ - candidate: inputs.repository, - staging: inputs.staging, - checkout: inputs.checkout, - source: inputs.source, - dirty: inputs.dirty, - receipt, - artifact, - ...emitted, - }); + try { + await runControls( + { + candidate: inputs.repository, + staging: inputs.staging, + checkout: inputs.checkout, + source: inputs.source, + dirty: inputs.dirty, + receipt, + artifact, + ...emitted, + }, + ran, + ); + } catch (error) { + // A harness self-check or an unexpected error stops the controls; the summary still names + // it, with only its fixed code, and every control that did not run is reported missing. + const line = productionFailure(error).at(-1); + ran.results.push({ + id: 'aborted', + expected: 'none', + observed: line.slice(line.lastIndexOf(': ') + 2), + }); + } return receipt; }, }, diff --git a/tools/cli-package/map-controls.test.mjs b/tools/cli-package/map-controls.test.mjs index dd9fb4d5..66e0912f 100644 --- a/tools/cli-package/map-controls.test.mjs +++ b/tools/cli-package/map-controls.test.mjs @@ -1,5 +1,10 @@ import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { copyFileSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { test } from 'node:test'; +import { fileURLToPath, pathToFileURL } from 'node:url'; import { FIXED_CONTROLS, cutInsideValue, @@ -201,3 +206,84 @@ test('the report prints one summary and returns 0 only for a passing run', () => assert.equal(empty.code, 1); assert.equal(empty.warned.length, 1); }); + +test('the required control set is exactly the seventeen the contract names', () => { + assert.deepEqual( + [...FIXED_CONTROLS], + [ + 'second-prepack', + 're-seal-receipt', + 're-seal-staging', + 're-recorded-baseline', + 'source-clean', + 'source-dirty', + 'sealed-dist', + 'dist-plain-chunk', + 'dist-rebuilt', + 'upstream-missing', + 'upstream-shape', + 'upstream-index', + 'upstream-cut-bytes', + 'upstream-cut-value', + 'upstream-shifted', + 'seal-verify', + 'restoration', + ], + ); + assert.equal(Object.isFrozen(FIXED_CONTROLS), true); +}); + +test('the entry point exits with the verdict and prints only fixed lines when the run fails', () => { + // The real entry point beside a stand-in core whose run returns or throws as each case needs; the + // report itself is the real one. + const here = (name) => fileURLToPath(new URL(`./${name}`, import.meta.url)); + const directory = mkdtempSync(join(tmpdir(), 'relavium-map-controls-entry-')); + try { + copyFileSync(here('map-controls.mjs'), join(directory, 'map-controls.mjs')); + copyFileSync(here('production-failure.mjs'), join(directory, 'production-failure.mjs')); + const outcome = (results) => ({ source: 'a'.repeat(40), dirty: false, results, chunks: 1 }); + const broken = passing(1); + broken[0] = { ...broken[0], observed: 'OTHER' }; + for (const [label, body, status, stderr] of [ + ['a passing run', `return ${JSON.stringify(outcome(passing(1)))};`, 0, /^$/u], + [ + 'a failed control', + `return ${JSON.stringify(outcome(broken))};`, + 1, + /^✗ CLI product map controls failed: second-prepack\n$/u, + ], + [ + 'a missing control', + `return ${JSON.stringify(outcome(passing(1).slice(1)))};`, + 1, + /^✗ CLI product map controls failed: second-prepack \(missing\)\n$/u, + ], + [ + 'a run that throws', + "throw new Error('PACKED_CLI_MAP_CONTROL_ORIGIN at /private/host/path');", + 1, + /^✗ CLI package production failed: PACKED_CLI_MAP_CONTROL_ORIGIN\n$/u, + ], + ]) { + writeFileSync( + join(directory, 'map-controls-core.mjs'), + `export { report } from ${JSON.stringify(pathToFileURL(here('map-controls-core.mjs')).href)};\n` + + `export async function runMapControls() {\n ${body}\n}\n`, + ); + const run = spawnSync(process.execPath, [join(directory, 'map-controls.mjs')], { + encoding: 'utf8', + timeout: 30_000, + }); + assert.equal(run.status, status, label); + assert.match(run.stderr, stderr, label); + // A completed run prints one summary line; a run that throws prints none. + assert.equal( + run.stdout.split('\n').filter(Boolean).length, + label === 'a run that throws' ? 0 : 1, + label, + ); + } + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); From 10993bb85179d0a05b20be563175b5eab1a56667 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sat, 24 Oct 2026 23:45:15 +0300 Subject: [PATCH 52/97] docs(repo): record the Step 9a.2b acceptance Step 9a.2b is independently accepted on development at 0f73c253 under the 2026-10-23 bar. Round 2, a new complete cumulative review of 436d7c67..0f73c253, reported no blocker, high or medium finding and no regression; the plan and correctness lenses were clean and the separate security sitting reported one low. Round 1 was not accepted: a regression in the author's own deferral record and an untested exit code, both corrected before round 2. - The register's 9a.2b row, the checkpoint and a new acceptance section; the plan's increment count now includes the native-identity increment (31). - deferred-tasks.md records round 2's lows (the workflow-level environment of the map-controls job, a narrowed chunk set, the refused-build dist removal) and amends the interrupted-runner entry; its freshness line names NI. - packaging.md's Status and interrupted-runner wording, the CLI reference row and the status lines in CLAUDE.md, AGENTS.md and current.md. NI is next, then Step 9b. This commit changes no code. Refs: ADR-0111, #46 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 +-- CLAUDE.md | 4 +-- docs/reference/cli/README.md | 2 +- docs/reference/cli/packaging.md | 6 ++-- docs/roadmap/current.md | 6 ++-- docs/roadmap/deferred-tasks.md | 31 ++++++++++++++++--- .../phase-2.5.5-hardening-and-remediation.md | 24 +++++++++++--- 7 files changed, 57 insertions(+), 20 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e039b805..5e88e8c5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,7 +117,7 @@ explicit; source CI does not certify later documentation commits. The joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now -accepted on `development`, below, as is Step 9a.2a; Step 9a.2b is implemented and under review; the native-identity increment NI and Steps 9b–19 have not started) and whole-wave acceptance remains Step 19. +accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI and Steps 9b–19 have not started) and whole-wave acceptance remains Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification @@ -130,7 +130,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-implementation-checkpoint--2026-10-24)). The native-identity increment NI (a Windows file-identity security fix the maintainer approved on 2026-10-24, register row NI) follows it, before Step 9b. `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a Windows file-identity security fix the maintainer approved on 2026-10-24, register row NI) is next, before Step 9b. `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. See [docs/roadmap/current.md](docs/roadmap/current.md) for live status. diff --git a/CLAUDE.md b/CLAUDE.md index 0b3482d4..92cf4612 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -206,7 +206,7 @@ explicit; source CI does not certify later documentation commits. The joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now -accepted on `development`, below, as is Step 9a.2a; Step 9a.2b is implemented and under review; the native-identity increment NI and Steps 9b–19 have not started) and whole-wave acceptance remains Step 19. +accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI and Steps 9b–19 have not started) and whole-wave acceptance remains Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification @@ -219,7 +219,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-implementation-checkpoint--2026-10-24)). The native-identity increment NI (a Windows file-identity security fix the maintainer approved on 2026-10-24, register row NI) follows it, before Step 9b. `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a Windows file-identity security fix the maintainer approved on 2026-10-24, register row NI) is next, before Step 9b. `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. For live status, per-PR history, milestone dates, and open obligations, see the canonical diff --git a/docs/reference/cli/README.md b/docs/reference/cli/README.md index 3867ce53..6144bb9c 100644 --- a/docs/reference/cli/README.md +++ b/docs/reference/cli/README.md @@ -13,5 +13,5 @@ Part of [reference/](../README.md). | [commands.md](commands.md) | The full `relavium` command surface (run, chat, chat-resume/list/export, agent run, list, create, import, export, logs, gate, init, provider) plus interactive TUI vs CI JSON mode. | | [home.md](home.md) | The bare-invocation interactive **Home** — the TTY gate, the read-only management strip over `history.db`, the Home↔chat mode machine + keymap, bracketed paste, the min-terminal degrade, and the signal/exit-code lifecycle (2.5.B / ADR-0054). | | [chat-session.md](chat-session.md) | The `relavium chat` agent-session REPL — entry, agent/model selection, the multi-turn loop, streaming, tool availability, the `--json` session-event stream, and exit code 4. | -| [packaging.md](packaging.md) | Wave 3 finite runtime graph, installed-package consumers and Step 8 native activation accepted on `development`; the Step 9a.1 candidate build and finalizer are accepted on `development` and Step 9a.2a's private manifest and symbolication are accepted on `development` and Step 9a.2b's product map controls and `package_test` receipt are implemented (acceptance pending); the remaining Step 9 custody and release gates remain staged. | +| [packaging.md](packaging.md) | Wave 3 finite runtime graph, installed-package consumers and Step 8 native activation accepted on `development`; the Step 9a.1 candidate build and finalizer are accepted on `development` and Step 9a.2a's private manifest and symbolication are accepted on `development` and Step 9a.2b's product map controls and `package_test` receipt are accepted on `development`; the remaining Step 9 custody and release gates remain staged. | | [agent-run-fixture.md](agent-run-fixture.md) | The `relavium agent run --fixture` cassette format — a `StreamChunk[][]` JSON recording of an agent run's LLM stream, for deterministic offline replay. | diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index af49b4dd..8b8c1c18 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -1,6 +1,6 @@ # CLI Packaging Contract -- **Status**: Compiled graph, installed package and Step 8 native activation independently accepted on `development`; the Step 9a.1 candidate build and finalizer are independently accepted on `development` (2026-10-24); Step 9a.2a's strict decoder, private manifest and symbolication are independently accepted on `development` (2026-10-24); Step 9a.2b's product map controls, post-receipt rebuild refusals and `package_test` receipt are implemented, with independent acceptance pending; the remaining Step 9 custody and release gates remain staged +- **Status**: Compiled graph, installed package and Step 8 native activation independently accepted on `development`; the Step 9a.1 candidate build and finalizer are independently accepted on `development` (2026-10-24); Step 9a.2a's strict decoder, private manifest and symbolication are independently accepted on `development` (2026-10-24); Step 9a.2b's product map controls, post-receipt rebuild refusals and `package_test` receipt are independently accepted on `development` (2026-10-24); the remaining Step 9 custody and release gates remain staged - **Date**: 2026-10-15 - **Related**: [ADR-0111](../../decisions/0111-cli-meta-operations-use-a-lazy-packaged-runtime.md), [commands.md](commands.md), [release runbook](../../runbooks/release-a-surface.md) @@ -487,8 +487,8 @@ package-test producer, adds the product map controls and only then flips the rec The runner keeps private maps and manifests longer than the producer does, always inside the producer's owner-only root; a runner killed outright can leave that root behind, the producer's -existing named limit, and an interrupted runner leaves its own empty temporary directory (see -deferred-tasks.md). The CI job runs on Linux only; a local run on Windows claims no POSIX mode-bit +existing named limit, and an interrupted runner can leave its own temporary directory, which may still +hold the baseline public tarball and its receipt (see deferred-tasks.md). The CI job runs on Linux only; a local run on Windows claims no POSIX mode-bit evidence, as above. ## Recipient custody diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index 96dc5138..f7d48f61 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -138,7 +138,7 @@ The [deferred register](deferred-tasks.md) retains its existing limits; closure **Wave 3 Steps 1–8 merged to `main` on 2026-10-18 as [PR #95](https://github.com/HodeTech/Relavium/pull/95), merge commit `6d2a7372` (22:44 UTC). Wave 3 itself is not complete: Steps 9–19 had not started at -that merge (Steps 9a.1 and 9a.2a are now accepted on `development` and Step 9a.2b is implemented and under review; the native-identity increment NI and Steps 9b–19 have not started) and whole-wave acceptance +that merge (Steps 9a.1, 9a.2a and 9a.2b are now accepted on `development`; the native-identity increment NI and Steps 9b–19 have not started) and whole-wave acceptance remains Step 19.** The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) joins the completed scope, every review attribution, the independent review's dispositions and the publication state observed at merge. It was the maintainer-authorized, scoped publication checkpoint. @@ -172,7 +172,7 @@ bar and round cap, as its medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, by the author alone). Its other findings are recorded in -[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is independently accepted on `development` at `cf97b7f1` (2026-10-24): its round 13 reported no blocker, high or medium finding and its security sitting was clean, and the round's low findings are recorded in deferred-tasks.md ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)); Step 9a.2 is split into 9a.2a and 9a.2b, and Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24) after three rounds, the third with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)); Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is implemented on `development` and under review ([checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-implementation-checkpoint--2026-10-24)), and the native-identity increment NI (a Windows file-identity security fix the maintainer approved on 2026-10-24, register row NI) follows it, before Step 9b; the substrate's +[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is independently accepted on `development` at `cf97b7f1` (2026-10-24): its round 13 reported no blocker, high or medium finding and its security sitting was clean, and the round's low findings are recorded in deferred-tasks.md ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)); Step 9a.2 is split into 9a.2a and 9a.2b, and Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24) after three rounds, the third with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)); Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24) after two rounds, the second with no blocker, high or medium finding and no regression ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)), and the native-identity increment NI (a Windows file-identity security fix the maintainer approved on 2026-10-24, register row NI) is next, before Step 9b; the substrate's 9a.1–9d increments carry a dedicated archive/crypto security sitting. Steps 10–19 follow the [increment table](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) and end with whole-wave acceptance at Step 19. `development` is pushed after each accepted increment; PR #95 no longer bounds the @@ -1141,7 +1141,7 @@ an ESLint selector rather than a list. ### Wave 3 — Clear the ground -**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Steps 9a.1 and 9a.2a accepted on `development` (2026-10-24), Step 9a.2b under review, NI and Steps 9b–19 NOT STARTED — 2026-10-24.** +**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Steps 9a.1, 9a.2a and 9a.2b accepted on `development` (2026-10-24), NI and Steps 9b–19 NOT STARTED — 2026-10-24.** ADR-0110–0113, their linked contracts and the nineteen-increment plan were approved on 2026-10-16. [Steps 1–8 were independently accepted](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) on `development`, with whole Step 8 reviewed at `c83c8575`, and published as the scoped checkpoint diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index 67759089..ff22d861 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -2,7 +2,7 @@ > Status: Living -> Last updated: 2026-10-24 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Step 9a.1 is independently accepted on `development` with its recorded low findings below, Step 9a.2a is independently accepted on `development` with its recorded low findings below, Step 9a.2b is implemented and under review, Steps 9b–19 not started; the PR95 increment is closed on `development` under the maintainer's 2026-10-21 round cap). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's edge cases with the maintainer's 2026-10-21 acceptance bar and round cap and the 2026-10-22 rulings. Existing approved deferrals remain. +> Last updated: 2026-10-24 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Step 9a.1 is independently accepted on `development` with its recorded low findings below, Step 9a.2a is independently accepted on `development` with its recorded low findings below, Step 9a.2b is independently accepted on `development` with its recorded low findings below, the native-identity increment NI (its approved entry below) and Steps 9b–19 not started; the PR95 increment is closed on `development` under the maintainer's 2026-10-21 round cap). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's edge cases with the maintainer's 2026-10-21 acceptance bar and round cap and the 2026-10-22 rulings. Existing approved deferrals remain. > The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) records completed prerequisites; full M/N lifecycle and other scheduled work remain with their owners. > > Earlier triage: the Phase-2.6 rewrite triaged every open item, the 2026-07-19 full-project @@ -733,9 +733,11 @@ for the two release entries. every test; the real runner shows the placement, but nothing asserts it. **Re-check:** a test that runs the runner's composition with recording ports (as `produce.test.mjs` does) and asserts every `scratch` path lies under the staging and the temporary root is gone afterwards. *(low · tools/cli-package/map-controls-core.mjs)* -- [ ] **An interrupted runner leaves its own empty temporary directory.** *Low.* *(rigor pass, PLAT-9A2B-6 = - SEC-9a2b-5 = INT-2)* The producer's interrupt handler empties the artifact directory and exits, so - `runMapControls`' `finally` never removes `relavium-map-controls-*/artifact`; it holds nothing private. +- [ ] **An interrupted runner can leave its own temporary directory.** *Low.* *(rigor pass, PLAT-9A2B-6 = + SEC-9a2b-5 = INT-2; round 2, CRIT-9A2B-R2-1)* The producer's interrupt handler empties the artifact directory and + exits, so `runMapControls`' `finally` never removes `relavium-map-controls-*/artifact`; and an interrupt after + `producePackage` returned, during the final identity check, leaves the baseline public tarball and its + `package_test` receipt in it. Nothing private. **Re-check:** remove the runner's root from the interrupt path too (for example with `exitOnInterrupt` around the run), with an interrupt test. *(low · tools/cli-package/map-controls-core.mjs)* - [ ] **The runner's own self-check refusals have no test.** *Low.* *(rigor pass, CRIT-9A2B-2)* @@ -755,6 +757,27 @@ for the two release entries. can enter it. **Re-check:** when Step 9d rebuilds the release graph, drop the cache or record why the release pack keeps it. *(low · .github/workflows/release.yml)* +Round 2, the acceptance round at `0f73c253` (no blocker, high or medium finding and no regression; the plan and +correctness lenses clean, the security sitting one low), reported seven verified findings. The plan's increment +count, the freshness line and the interrupted-runner wording are corrected in the acceptance commit, which changes +no code (INT-9A2B-R2-1, INT-9A2B-R2-2, CRIT-9A2B-R2-1, the last also amending the entry above); these are recorded. + +- [ ] **The workflow-level environment of the map-controls job is not pinned.** *Low.* *(round 2, SEC-9A2B-R2-1 = + TCA-9A2B-R2-3)* `checkMapControls` pins the job's own `env` but not ci.yml's top-level `env`, which every job + inherits, nor a job `container` or `services` key; a workflow-wide `NODE_OPTIONS` preload or a secret would reach + the controls step with every pin passing. **Re-check:** pin `Object.keys(workflow.env)` to the two Turbo keys the + job overrides, assert `job.container` and `job.services` are undefined, and add the negative cases. + *(low · tools/ci/require-packed-cli.test.mjs)* +- [ ] **A narrowed set of anchor-bearing chunks goes unnoticed.** *Low.* *(round 2, TCA-9A2B-R2-1)* The verdict + derives the per-chunk ids from the runner's own chunk count, required only to be at least one, so a regression + that drops a chunk (for example `.slice(0, 1)` on the derivation) leaves the runner green with 21 controls. + **Re-check:** derive the expected count independently — every ANCHORS source covered by at least one controlled + chunk — or test the composition with recording ports. *(low · tools/cli-package/map-controls-core.mjs)* +- [ ] **Removing a refused control build's `dist` has no test.** *Low.* *(round 2, TCA-9A2B-R2-2)* Deleting the + `rmSync` in `pack()` passes everything; the bundler's maps would stay in the candidate's `dist` (inside the + producer's owner-only root) until the next build. **Re-check:** with the composition test of the F5 entry, assert + `dist` holds no `.map` file after each refused control build. *(low · tools/cli-package/map-controls-core.mjs)* + ## Native file identity on Windows — approved increment (2026-10-24) Found by Step 9a.2a's push CI (the advisory Windows job of run 40105286543). The maintainer approved a separate diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 1401daa5..ab289efc 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -3700,7 +3700,7 @@ No credential values or request captures belong in tracked evidence. | 9a.1 | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — candidate build and finalizer:** an owned isolated candidate copy, fresh uncached upstream maps, candidate-mode tsup selection with absent `sourcesContent`, the prepack finalizer (upstream and final map validation, repository-relative sources, exact trailer removal before hashes), owner-only private staging outside public artifacts and caches, closure/seal adaptation, the ordinary map-free control, the PR #95 archive/worker fold-ins, and a fail-closed `release.yml` publish guard that refuses any non-release candidate until 9d and Wave 4b custody exist. The receipt stays `compiled_package_test`. *Implemented on `development` on 2026-10-22 ([checkpoint](#wave-3-step-9a1-implementation-checkpoint--2026-10-22)); independently accepted at `cf97b7f1` on 2026-10-24 ([acceptance](#wave-3-step-9a1-acceptance--2026-10-24)).* | | 9a.2 | **Recorded split — manifest, symbolication and package-test receipt:** the private map manifest keyed by `dist/.js`, actual packed CLI and engine frame symbolication against the matching source revision, wrong JS/map/source and missing/malformed-upstream negative controls (product copies via `check:map-controls`, also a required uncached CI job), and the atomic `package_test` receipt flip across seal, binding, consumers, aggregate and both workflows. *Split on 2026-10-24 (maintainer-approved; ID coverage #46/ADR-0111 unchanged) into 9a.2a and 9a.2b below, which replace it as increments ([preflight decisions](#wave-3-step-9a2-preflight-decisions--2026-10-24)).* | | 9a.2a | **Recorded split, 2026-10-24 (maintainer-approved) — decoder, private manifest and symbolication:** an in-house strict source-map decoder; strict mapping and range validation of upstream and final maps in the finalizer; the private map manifest (`staging/manifest.json`, keyed by `dist/.js`, with a per-source digest inventory and the commit's tree), sealed after the archive inspection; real packed CLI and `@relavium/core` frame capture through a tooling-only preload during an offline `relavium import`, run on the finalized `dist` that the inspection proves byte-equal to the tarball; symbolication of those frames against the matching source revision (clean: the checkout's Git objects; dirty, local only: the candidate copy), wired into the seal before the receipt is written. The receipt stays `compiled_package_test`. Security sitting. *Independently accepted on `development` at `a92a7602` on 2026-10-24 ([checkpoint](#wave-3-step-9a2a-implementation-checkpoint--2026-10-24), [acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)).* | -| 9a.2b | **Recorded split, 2026-10-24 (maintainer-approved) — product map controls and the package-test receipt:** `check:map-controls` over product copies (wrong JS by line and column, wrong map, permuted sources, wrong source, missing, malformed and truncated upstream maps, a shifted valid upstream map, and the package-test producer's post-receipt rebuild refusals that ADR-0111 lists), run in `pnpm run ci` and as a required uncached CI job inside `ci`'s needs (the required contexts stay `ci` and `coverage`); then the atomic `package_test` flip across seal, binding, consumers, aggregate and both workflows, version 1 kept. Security sitting. *Preflight decisions approved on 2026-10-24 ([decisions](#wave-3-step-9a2b-preflight-decisions--2026-10-24)); implemented on `development` on 2026-10-24, acceptance pending ([checkpoint](#wave-3-step-9a2b-implementation-checkpoint--2026-10-24)).* | +| 9a.2b | **Recorded split, 2026-10-24 (maintainer-approved) — product map controls and the package-test receipt:** `check:map-controls` over product copies (wrong JS by line and column, wrong map, permuted sources, wrong source, missing, malformed and truncated upstream maps, a shifted valid upstream map, and the package-test producer's post-receipt rebuild refusals that ADR-0111 lists), run in `pnpm run ci` and as a required uncached CI job inside `ci`'s needs (the required contexts stay `ci` and `coverage`); then the atomic `package_test` flip across seal, binding, consumers, aggregate and both workflows, version 1 kept. Security sitting. *Preflight decisions approved on 2026-10-24 ([decisions](#wave-3-step-9a2b-preflight-decisions--2026-10-24)); independently accepted on `development` at `0f73c253` on 2026-10-24 ([checkpoint](#wave-3-step-9a2b-implementation-checkpoint--2026-10-24), [acceptance](#wave-3-step-9a2b-acceptance--2026-10-24)).* | | NI | **Native file identity on Windows (maintainer-approved 2026-10-24; after 9a.2b, before 9b):** every file and directory identity check compares `dev`/`ino` read with `{ bigint: true }` — the filesystem policy's pinned roots, the workflow catalog's open check and the media store's pins, read check and temporary claims — so a replaced file or directory can never compare equal on Windows, where `ino` is a 64-bit NTFS file ID that a JavaScript number cannot hold exactly; a replacement test per site. Found by 9a.2a's push CI ([acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)); recorded in [deferred-tasks.md](../deferred-tasks.md#native-file-identity-on-windows--approved-increment-2026-10-24). Security sitting. | | 9b | **Recorded split — recipient custody:** synthetic-only isolated GnuPG controls (public-only home/keyring, pinned primary and encryption-subkey fingerprints, machine-readable refusal of missing, expired, revoked or unusable recipients), encrypt once and reuse the original ciphertext. Explicit crypto/custody security sitting. Fixture proof never claims production recipient or custody configuration. | | 9c | **Recorded split — offline release state controller:** permission-enforcing offline GitHub/npm controller, permissioned prebuild discovery and reservation, and every rerun, partial, conflict and expiry state, with no overwrite, rebuild or re-encryption on uncertain history. | @@ -9398,7 +9398,7 @@ plan amendment (row PR95) and the recorded splits; ID coverage is unchanged apar | --- | --- | | Acceptance | Each increment keeps the existing rule: implement, author gates, commit, then NEW independent complete cumulative review rounds until two consecutive rounds are clean after the last material fix. Comment- or documentation-only nits do not reset the count. Security-touching increments add a separate scoped sitting: PR95, 9a.1, 9a.2, 9b, 9c, 9d (whole Step 9 custody), 10a, 10c, 11b, 13c, 14b, 15c, 16a, 16b, 16c (whole Step 16), 18 and 19b (whole wave). PR95 instead follows the maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)). **Amended 2026-10-23 (maintainer, after Step 9a.1's twelfth round):** from Step 9a.1's round 13 on, every increment is accepted after one NEW independent complete cumulative round that reports no blocker, high or medium finding and no regression; that round's low findings are recorded in [deferred-tasks.md](../deferred-tasks.md) with their file, a minimal example and a re-check note rather than fixed in the loop, and nits are optional. A high or medium finding is still corrected and followed by a new round, and the separate security sittings above are unchanged. | | Publication | `development` is pushed after each accepted increment, so its actual CI (including the four same-tarball platform consumers) is observed per increment. The Wave 3 PR follows Step 19. | -| Plan shape | PR95 + 9a.1, 9a.2, 9b, 9c, 9d + 10a–c + 11a–b + 12a–b + 13a–c + 14a–b + 15a–c + 16a–c + 17a–b + 18 + 19a–b (29 increments). Each split is recorded before its step starts. **Amended 2026-10-24 (maintainer):** 9a.2 is split into 9a.2a and 9a.2b, so the plan has 30 increments ([9a.2 preflight decisions](#wave-3-step-9a2-preflight-decisions--2026-10-24)). | +| Plan shape | PR95 + 9a.1, 9a.2, 9b, 9c, 9d + 10a–c + 11a–b + 12a–b + 13a–c + 14a–b + 15a–c + 16a–c + 17a–b + 18 + 19a–b (29 increments). Each split is recorded before its step starts. **Amended 2026-10-24 (maintainer):** 9a.2 is split into 9a.2a and 9a.2b, so the plan has 30 increments ([9a.2 preflight decisions](#wave-3-step-9a2-preflight-decisions--2026-10-24)). **Amended 2026-10-24 (maintainer, E6):** the native-identity increment NI is added after 9a.2b, so the plan has 31 increments ([9a.2a acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)). | | Reassignments | G54 moves from row 17 to 11a; #311's implementation moves to 10b and its certification stays in 17a; G36/G46 are re-scoped in 10a to the branch that replaces the raw-stack branch and are certified in 17b; promotion of the Steps 6–10 commands.md sections moves from Step 19 to 10c. | | Interim release guard | 9a.1 adds a fail-closed check to the `release.yml` publish job that refuses any non-release candidate. Consequence, acknowledged: no npm publication, including a v0.1.x patch, until 9d and Wave 4b production custody exist. | | PR95 scrubber residuals | Fixed before 9a: the `api-key:Bearer`/`Basic` leak; raw `?`/`#` userinfo masked unless the text before it is host-shaped and an `=` assignment separates it from the authority `@`, in both the LLM leaf and core's twin predicate (core's other documented differences are recorded, not removed); an extended finite scheme list that masks colon-bearing userinfo for the new schemes; the scanner invariants, prefilter and witnesses; the load-sensitive 8 MiB timing case. LOW-1 and LOW-2 become approved deferrals with their recorded triggers. | @@ -9871,6 +9871,20 @@ build that ran the bundler has its `dist` removed; the seventeen fixed controls the file" wording excepts `dist`; the checkpoint's counts (25 controls, 932 tests, four lows and one medium) and the 9a.2a push-CI job count (fourteen required jobs) are corrected; and the status lines name the NI increment. -**Remaining gates.** Round 2 — one NEW complete cumulative round with no blocker, high or medium finding and no -regression, with the separate scoped security sitting at the same head and bar — then the full `pnpm run ci` and -the `map-controls` job green on the push CI. +Round 2 accepted the increment; see the acceptance below. + +### Wave 3 Step 9a.2b acceptance — 2026-10-24 + +**Step 9a.2b is independently accepted on `development` at `0f73c253` (2026-10-24)** under the Acceptance row as +amended on 2026-10-23. Round 2, a NEW independent complete cumulative review of `436d7c67..0f73c253` with every +reviewer in its own worktree detached at that head, reported no blocker, high or medium finding and no regression; +its separate security sitting, run at the same head as its own reviewer and report, reported one low; the plan and +correctness lenses were clean. The plan reviewer checked the commit order (the flip never exists in a tree without +the controls, the required job and its guard before it), ran the real runner at the reviewed head (25 of 25 +expected controls on a clean tree) and showed that removing the artifact half of `PACKED_CLI_ALREADY_SEALED` fails +both re-seal controls. The round's seven verified findings (five low, two nits) are recorded in +[deferred-tasks.md](../deferred-tasks.md#wave-3-step-9a2b--recorded-low-findings-2026-10-24) or corrected in the +acceptance commit, which changes no code: the plan's increment count now includes NI, deferred-tasks.md's freshness +line names it, and the contract and the interrupted-runner record say that directory can still hold the baseline +public tarball and receipt. The receipt now records `package_test`; the native-identity increment NI is next, then +Step 9b. From 451223f7956f1aec52cf55d857bc793b2bed1380 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 01:19:47 +0300 Subject: [PATCH 53/97] docs(repo): record the NI preflight decisions A read-only preflight at 10993bb8 mapped every file and directory identity comparison. The maintainer approved each recommendation on 2026-10-25: a bigint projection in the media store, five permanent precision controls, a Windows-only probe with repeated replacement tests, the three same-class tooling sites fixed in NI, an unsigned claim token and ReFS recorded as a scoped limit. The verified facts (signed BigInt64Array stats, overlayfs xino high bits, ReFS 64-bit identifiers not unique) are recorded with them. Refs: ADR-0105, ADR-0106 Co-Authored-By: Claude Opus 5.5 --- .../phase-2.5.5-hardening-and-remediation.md | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index ab289efc..c3fd02ae 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -9888,3 +9888,42 @@ acceptance commit, which changes no code: the plan's increment count now include line names it, and the contract and the interrupted-runner record say that directory can still hold the baseline public tarball and receipt. The receipt now records `package_test`; the native-identity increment NI is next, then Step 9b. + +### Wave 3 NI preflight decisions — 2026-10-25 + +A read-only preflight (three readers and a synthesis at `10993bb8`; the detailed record and a Turkish decision +register are local and Git-ignored) mapped every file and directory identity comparison in the tree. The maintainer +approved each recommendation on 2026-10-25; no new ADR or runtime dependency follows, and ADR-0105 and ADR-0106 +receive dated correction notes after the precedent of ADR-0106's 2026-10-13 exact temporary claim identity +correction. The sites are the row's five files: the filesystem policy's canonical and lexical root pins; the +workflow catalog's open check, the only swap guard on Windows, where `O_NOFOLLOW` is undefined; and the media +store's root pin, its file snapshot and owned-file check (the authority to delete), its read check, its publish +handle and the temporary sweep's claim directory, child, marker and moved-blob checks with the `Number` cross-check +inside `nativeFile`. One test precondition reads number identities as well. No persisted format changes: +`media-claim-v2` names already carry exact base-36 tokens, and the v1 reader admits only safe tokens, whose bigint +form is identical. + +Three facts were verified after the preflight. Node 22 fills bigint stats from a signed `BigInt64Array`, so a 64-bit +file ID with its top bit set (an NTFS sequence number of 0x8000 or more) reads negative; exact equality is +unaffected, but the claim token refused it and such a temporary was retained forever. The defect is not +Windows-only: Linux overlayfs's `xino` feature "uses the high inode number bits for fsid" (kernel documentation). +And on ReFS, including Dev Drive, Microsoft documents that the 64-bit file identifier "is not guaranteed to be +unique" (ReFS uses 128-bit identifiers). + +| Decision | Ruling | +| --- | --- | +| Snapshot shape | The media store reads each identity with one bigint stat and projects it once into an internal record: bigint `dev` and `ino`, number `nlink`, `size` and `mtimeMs` (from `mtimeNs`, so today's sub-millisecond values stay), and the type predicates of the same stat. A path is never stat'ed twice for one snapshot. The CLI sites keep `BigIntStats`, since no other field there meets an identity check. | +| Permanent proof | Five filesystem causal controls, one per site: the policy pins, the catalog open check, the root pin, the owned-file check and the temporary sweep's `same`. Each mutant returns that site's whole comparison to `Number` and is witnessed by a real replacement under injected identities above 2^60, where adjacent numbers are 256 apart. The read check, the `nativeFile` cross-check and the per-read variants are covered by the recorded mutation sweep and the type checks. | +| Windows evidence | Acceptance rests on the deterministic proof. `fs-policy.test.ts` gains a Windows-only probe (about 200 rename-and-`mkdir` pairs, each distinct as bigints, logging how many collide as numbers or read negative) and `{ repeats: 20 }` on its real replacement tests. The Windows clause closes after three consecutive green advisory Windows runs on pushed heads, with the logged counts recorded; the existing Windows steps are unchanged. | +| Tooling sites | The three same-class tooling checks are fixed in NI after the `tools/cli-package/files.mjs` pattern, each with an injected-identity check: the overflow capture's destination check, the budget-replay evidence retention's directory identity (it deletes recursively) and that tool's lifecycle-smoke precondition. | +| Read check | Converted for uniformity. Its second-stat comparison is tested with a faked second stat and recorded as a consistency check that a real filesystem cannot fail; its `dev` comparisons are exact on every platform. No permanent control. | +| Sign and ReFS | `nativeIdentityToken` maps each value with `BigInt.asUintN(64, …)`, in one place: tokens compare with tokens, and every value without its top bit set keeps a byte-identical token, so existing names stay compatible. A test shows that a negative identity can be claimed. ReFS is recorded in deferred-tasks.md as a scoped limit, since a fix needs native code or a dependency. The claim becomes exact identity on filesystems whose 64-bit file indexes are unique, such as NTFS, ext4 and APFS. | + +Engineering defaults recorded with the rulings: +- No identity helper is shared across packages, and the shards get no pin. +- `nativeFile` keeps its second `lstat`. +- The temporary sweep's token comparison line stays byte-identical, since its precision control anchors on it. +- New witnesses go into new test files, so no pinned count moves. +- `CAS_COOPERATIVE_READ` is the one anchor that drifts; it is updated in the same commit. +- No identity value enters an error, event, log or JSON. +- ADR-0105's cooperative-parent assumption is unchanged. From 477e11dd3f823b1c53a876c35b62d1aa4e33fa86 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 01:26:15 +0300 Subject: [PATCH 54/97] fix(cli): pin filesystem roots and admit catalog opens by exact native identity The filesystem policy pinned each root's canonical and lexical dev/ino as JavaScript numbers, and the workflow catalog compared its admitting stat and its open by number too. A 64-bit file ID (NTFS on Windows, overlayfs xino on Linux) exceeds a Number's 53 bits, so a replaced directory or file could compare equal; on Windows the catalog comparison is the only swap guard, since O_NOFOLLOW is undefined there. Both sites now read with { bigint: true } and the pin fields are bigint, so a read without the option no longer typechecks. A new witness file substitutes every native identity with 2^60 + k (distinct as bigints, one Number) and proves a real replacement is refused through the canonical read, the lexical read and the catalog open. The fs-invocation precondition reads exactly too. Refs: ADR-0105, NI Co-Authored-By: Claude Opus 5.5 --- apps/cli/src/commands/fs-invocation.test.ts | 4 +- apps/cli/src/engine/tool-host/fs-policy.ts | 13 +- .../engine/tool-host/native-identity.test.ts | 187 ++++++++++++++++++ apps/cli/src/workflows/catalog.ts | 8 +- 4 files changed, 201 insertions(+), 11 deletions(-) create mode 100644 apps/cli/src/engine/tool-host/native-identity.test.ts diff --git a/apps/cli/src/commands/fs-invocation.test.ts b/apps/cli/src/commands/fs-invocation.test.ts index 71d7cb7d..1574e977 100644 --- a/apps/cli/src/commands/fs-invocation.test.ts +++ b/apps/cli/src/commands/fs-invocation.test.ts @@ -519,10 +519,10 @@ it('revalidates a revoked policy before binding an agent or connecting MCP durin ); if (options === undefined) throw new Error('seed builder was not reached'); // Keep the admitted directory alive so Linux cannot recycle its inode for the replacement. - const original = statSync(outside); + const original = statSync(outside, { bigint: true }); renameSync(outside, `${outside}-old`); mkdirSync(outside); - const replacement = statSync(outside); + const replacement = statSync(outside, { bigint: true }); expect([replacement.dev, replacement.ino]).not.toEqual([original.dev, original.ino]); const startMcpClient = vi .fn>() diff --git a/apps/cli/src/engine/tool-host/fs-policy.ts b/apps/cli/src/engine/tool-host/fs-policy.ts index 0269c0a2..77d3f89e 100644 --- a/apps/cli/src/engine/tool-host/fs-policy.ts +++ b/apps/cli/src/engine/tool-host/fs-policy.ts @@ -15,10 +15,10 @@ export class FsPolicyError extends HostDeniedError { interface DirectoryPin { readonly lexical: string; readonly canonical: string; - readonly dev: number; - readonly ino: number; - readonly lexicalDev: number; - readonly lexicalIno: number; + readonly dev: bigint; + readonly ino: bigint; + readonly lexicalDev: bigint; + readonly lexicalIno: bigint; readonly linked: boolean; } @@ -59,8 +59,9 @@ function pinDirectory(path: string): DirectoryPin { // Match fs/promises.realpath's native canonicalization. The legacy synchronous // walker preserves Windows 8.3 spellings, which are not equal to their long names. const canonical = realpathSync.native(lexical); - const stat = statSync(canonical); - const entry = lstatSync(lexical); + // Exact native identities: a 64-bit file ID (NTFS, overlayfs xino) exceeds a Number's precision. + const stat = statSync(canonical, { bigint: true }); + const entry = lstatSync(lexical, { bigint: true }); if (!stat.isDirectory()) throw new FsPolicyError(); return Object.freeze({ lexical, diff --git a/apps/cli/src/engine/tool-host/native-identity.test.ts b/apps/cli/src/engine/tool-host/native-identity.test.ts new file mode 100644 index 00000000..bebb2537 --- /dev/null +++ b/apps/cli/src/engine/tool-host/native-identity.test.ts @@ -0,0 +1,187 @@ +import { + lstatSync, + mkdirSync, + mkdtempSync, + realpathSync, + renameSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, + type BigIntStats, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { discoverCatalog } from '../../workflows/catalog.js'; +import { + admitInvocationFsGrants, + FsPolicyError, + fsPolicyScopeRoots, + selectInvocationFsPolicy, +} from './fs-policy.js'; + +/** + * POSIX inodes stay far below 2^53, so these cases substitute every identity the code reads: each real + * (dev, ino) becomes 2^60 + k. Distinct substitutes stay distinct as bigints but share one Number (the + * spacing there is 256), so a read without `{ bigint: true }` or a comparison through Number lets a real + * replacement compare equal, as a 64-bit Windows file ID can. + */ +const identities = vi.hoisted(() => { + const state: { fakes: Map; beforeOpen: (() => void) | undefined } = { + fakes: new Map(), + beforeOpen: undefined, + }; + return state; +}); +vi.mock('node:fs', async (original) => { + const native = await original(); + const substitute = (info: unknown, exact: BigIntStats | undefined): void => { + if (exact === undefined || typeof info !== 'object' || info === null || !('ino' in info)) + return; + const key = `${exact.dev}:${exact.ino}`; + const value = identities.fakes.get(key) ?? (1n << 60n) + BigInt(identities.fakes.size); + identities.fakes.set(key, value); + Object.defineProperty(info, 'ino', { + value: typeof info.ino === 'bigint' ? value : Number(value), + }); + }; + return { + ...native, + statSync: (...args: Parameters) => { + const info = native.statSync(...args); + substitute(info, native.statSync(args[0], { bigint: true, throwIfNoEntry: false })); + return info; + }, + lstatSync: (...args: Parameters) => { + const info = native.lstatSync(...args); + substitute(info, native.lstatSync(args[0], { bigint: true, throwIfNoEntry: false })); + return info; + }, + fstatSync: (...args: Parameters) => { + const info = native.fstatSync(...args); + substitute(info, native.fstatSync(args[0], { bigint: true })); + return info; + }, + openSync: (...args: Parameters) => { + const before = identities.beforeOpen; + identities.beforeOpen = undefined; + before?.(); + return native.openSync(...args); + }, + }; +}); + +let base: string; +let workspace: string; +beforeEach(() => { + identities.fakes.clear(); + base = realpathSync.native(mkdtempSync(join(tmpdir(), 'relavium-native-identity-'))); + workspace = join(base, 'workspace'); + mkdirSync(workspace); +}); +afterEach(() => { + identities.beforeOpen = undefined; + rmSync(base, { recursive: true, force: true }); +}); + +/** The fixture must make the original and its replacement collide as Numbers and only as Numbers. */ +function expectNumberCollision(original: bigint, replacement: bigint): void { + expect(replacement).not.toBe(original); + expect(Number(replacement)).toBe(Number(original)); +} + +function refused(check: () => unknown): boolean { + try { + check(); + return false; + } catch (error) { + if (error instanceof FsPolicyError) return true; + throw error; + } +} + +describe('filesystem policy pins compare exact native identities', () => { + it('refuses a replaced selected directory whose identity collides as a Number', () => { + const grant = join(base, 'grant'); + const selected = join(grant, 'selected'); + mkdirSync(selected, { recursive: true }); + const policy = selectInvocationFsPolicy(admitInvocationFsGrants([grant]), workspace, [ + selected, + ]); + const original = statSync(selected, { bigint: true }).ino; + // Keep the original alive so the platform cannot recycle its inode for the replacement. + renameSync(selected, `${selected}-old`); + mkdirSync(selected); + expectNumberCollision(original, statSync(selected, { bigint: true }).ino); + expect( + refused(() => fsPolicyScopeRoots(policy, workspace, 'project')), + 'FS_ROOT_IDENTITY_PRECISION', + ).toBe(true); + }); + + it('refuses an unchanged alias whose target was replaced (the canonical identity)', () => { + const target = join(base, 'target'); + const alias = join(base, 'alias'); + mkdirSync(target); + symlinkSync(target, alias, 'junction'); + const policy = selectInvocationFsPolicy(admitInvocationFsGrants([alias]), workspace, undefined); + const original = statSync(target, { bigint: true }).ino; + const entry = lstatSync(alias, { bigint: true }).ino; + renameSync(target, `${target}-old`); + mkdirSync(target); + expectNumberCollision(original, statSync(target, { bigint: true }).ino); + expect(lstatSync(alias, { bigint: true }).ino).toBe(entry); + expect( + refused(() => fsPolicyScopeRoots(policy, workspace, 'project')), + 'FS_ROOT_IDENTITY_PRECISION', + ).toBe(true); + }); + + it('refuses a replaced alias whose target is unchanged (the lexical identity)', () => { + const target = join(base, 'target'); + const alias = join(base, 'alias'); + mkdirSync(target); + symlinkSync(target, alias, 'junction'); + const policy = selectInvocationFsPolicy(admitInvocationFsGrants([alias]), workspace, undefined); + const canonical = statSync(target, { bigint: true }).ino; + const original = lstatSync(alias, { bigint: true }).ino; + renameSync(alias, `${alias}-old`); + symlinkSync(target, alias, 'junction'); + expectNumberCollision(original, lstatSync(alias, { bigint: true }).ino); + expect(statSync(target, { bigint: true }).ino).toBe(canonical); + expect( + refused(() => fsPolicyScopeRoots(policy, workspace, 'project')), + 'FS_ROOT_IDENTITY_PRECISION', + ).toBe(true); + }); +}); + +describe('the workflow catalog admits an open by exact native identity', () => { + const workflow = (id: string): string => + `schema_version: '1.0'\nworkflow:\n id: ${id}\n nodes:\n - { id: a, type: input }\n - { id: b, type: output }\n edges:\n - { from: a, to: b }\n`; + + it('refuses a catalog file swapped between its admission and its open', () => { + const configDir = join(base, '.relavium'); + const file = join(configDir, 'workflows', 'hello.yaml'); + mkdirSync(join(configDir, 'workflows'), { recursive: true }); + writeFileSync(file, workflow('hello')); + // The swap lands after the catalog's admitting stat and before its open. + identities.beforeOpen = () => { + renameSync(file, `${file}-old`); + writeFileSync(file, workflow('swapped')); + }; + const entries = discoverCatalog({ projectConfigDir: configDir, cwd: base, kind: 'workflows' }); + expect(identities.beforeOpen).toBeUndefined(); + expectNumberCollision( + statSync(`${file}-old`, { bigint: true }).ino, + statSync(file, { bigint: true }).ino, + ); + expect( + entries.map((entry) => ({ valid: entry.valid, error: entry.error })), + 'CATALOG_IDENTITY_PRECISION', + ).toEqual([{ valid: false, error: 'not the admitted regular file' }]); + }); +}); diff --git a/apps/cli/src/workflows/catalog.ts b/apps/cli/src/workflows/catalog.ts index fa57e325..1b573a7e 100644 --- a/apps/cli/src/workflows/catalog.ts +++ b/apps/cli/src/workflows/catalog.ts @@ -126,7 +126,7 @@ function readEntry(path: string, cwd: string, kind: CatalogKind, root: string): const canonical = realpathSync.native(path); if (!isWithinFsRoot(root, canonical)) return invalidEntry(path, rel, 'outside the project catalog'); - const stats = statSync(canonical); + const stats = statSync(canonical, { bigint: true }); // statSync FOLLOWS a symlink, so a `.yaml` symlinked to a directory, FIFO, or character device (e.g. // /dev/zero) resolves here — and readFileSync on a FIFO/char-device BLOCKS INDEFINITELY (a local DoS via a // crafted symlink in the catalog dir). Reject anything that is not a regular file BEFORE reading; the size @@ -138,7 +138,7 @@ function readEntry(path: string, cwd: string, kind: CatalogKind, root: string): // bytes/char, so `chars * 4` is a byte ceiling that never false-rejects a within-limit file (a dense // multibyte file just under the char cap is not wrongly hidden) while still bailing before we slurp a // genuinely huge file into memory. The parser then re-applies the exact char cap. - if (stats.size > MAX_SOURCE_CHARS * 4) { + if (stats.size > BigInt(MAX_SOURCE_CHARS * 4)) { return invalidEntry(path, rel, 'exceeds the size limit'); } // Contained aliases are allowed; open their canonical target without following a newly planted final link. @@ -147,7 +147,9 @@ function readEntry(path: string, cwd: string, kind: CatalogKind, root: string): constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, ); try { - const opened = fstatSync(fd); + // Exact bigint identities: Windows has no O_NOFOLLOW, so this is its only swap guard, and its + // 64-bit file IDs exceed a Number's precision. + const opened = fstatSync(fd, { bigint: true }); if (!opened.isFile() || opened.dev !== stats.dev || opened.ino !== stats.ino) { return invalidEntry(path, rel, 'not the admitted regular file'); } From 2dc469d4f88b9fac9b6fe7094b417dda8aa4535d Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 01:26:15 +0300 Subject: [PATCH 55/97] test(cli): probe Windows identity collisions and repeat the replacement tests A Windows-only probe makes 200 rename-and-mkdir pairs, asserts each pair is distinct as exact identities and logs how many collide as Numbers or read negative, so the advisory Windows job measures the diagnosed mechanism on the runner itself. The real replacement tests repeat twenty times. Refs: NI Co-Authored-By: Claude Opus 5.5 --- .../src/engine/tool-host/fs-policy.test.ts | 99 +++++++++++++------ 1 file changed, 69 insertions(+), 30 deletions(-) diff --git a/apps/cli/src/engine/tool-host/fs-policy.test.ts b/apps/cli/src/engine/tool-host/fs-policy.test.ts index eea82e41..53269284 100644 --- a/apps/cli/src/engine/tool-host/fs-policy.test.ts +++ b/apps/cli/src/engine/tool-host/fs-policy.test.ts @@ -7,6 +7,7 @@ import { realpathSync, renameSync, rmSync, + statSync, symlinkSync, writeFileSync, } from 'node:fs'; @@ -79,6 +80,9 @@ beforeEach(() => { }); afterEach(() => rmSync(base, { recursive: true, force: true })); +// A real replacement could compare equal one run in sixty on Windows before identities were exact (NI). +const REPLACEMENT_REPEATS = { repeats: 20 }; + function capability(policy: FsPathPolicy, tier: FsScopeTier = 'project') { return createNodeFsCapability({ tier, @@ -160,6 +164,7 @@ describe('invocation authority and owned native identities', () => { it.each(['directory', 'alias'] as const)( 'revokes a replaced %s after admission, before the first tool call', + REPLACEMENT_REPEATS, async (kind) => { const root = kind === 'directory' ? grant : join(base, 'alias'); if (kind === 'alias') symlinkSync(grant, root, 'junction'); @@ -180,24 +185,30 @@ describe('invocation authority and owned native identities', () => { }, ); - it('pins workspace and selected-subroot identities and owns caller arrays/config', async () => { - const roots = [grant]; - const input = { - tier: 'project' as const, - workspaceDir: workspace, - readOnly: true, - extraRoots: roots, - }; - const fs = createNodeFsCapability(input); - roots.length = 0; - input.readOnly = false; - expect((await fs.readFile('../grant/ok.txt', {})).content).toBe('grant'); - await expect(fs.writeFile('new.txt', 'denied', {})).rejects.toThrow(/read.only|not available/); - renameSync(workspace, `${workspace}-old`); - mkdirSync(workspace); - writeFileSync(join(workspace, 'ok.txt'), 'replacement'); - await expect(fs.readFile('ok.txt', {})).rejects.toThrow(/pinned identities/); - }); + it( + 'pins workspace and selected-subroot identities and owns caller arrays/config', + REPLACEMENT_REPEATS, + async () => { + const roots = [grant]; + const input = { + tier: 'project' as const, + workspaceDir: workspace, + readOnly: true, + extraRoots: roots, + }; + const fs = createNodeFsCapability(input); + roots.length = 0; + input.readOnly = false; + expect((await fs.readFile('../grant/ok.txt', {})).content).toBe('grant'); + await expect(fs.writeFile('new.txt', 'denied', {})).rejects.toThrow( + /read.only|not available/, + ); + renameSync(workspace, `${workspace}-old`); + mkdirSync(workspace); + writeFileSync(join(workspace, 'ok.txt'), 'replacement'); + await expect(fs.readFile('ok.txt', {})).rejects.toThrow(/pinned identities/); + }, + ); it('returns reusable extra-root paths through write, glob and mention without a doubled anchor', async () => { const fs = capability( @@ -520,18 +531,22 @@ it.each(['workspace', 'grant', 'selector'] as const)( }, ); -it('revokes a selected subdirectory replacement even when its enclosing grant is unchanged', async () => { - const selected = join(grant, 'selected'); - mkdirSync(selected); - const fs = capability( - selectInvocationFsPolicy(admitInvocationFsGrants([grant]), workspace, [selected]), - ); - renameSync(selected, selected + '-old'); - mkdirSync(selected); - writeFileSync(join(selected, 'a.txt'), 'replacement'); - await expect(fs.readFile(join(selected, 'a.txt'), {})).rejects.toThrow(/pinned identities/); - expect(observed.open).not.toHaveBeenCalled(); -}); +it( + 'revokes a selected subdirectory replacement even when its enclosing grant is unchanged', + REPLACEMENT_REPEATS, + async () => { + const selected = join(grant, 'selected'); + mkdirSync(selected); + const fs = capability( + selectInvocationFsPolicy(admitInvocationFsGrants([grant]), workspace, [selected]), + ); + renameSync(selected, selected + '-old'); + mkdirSync(selected); + writeFileSync(join(selected, 'a.txt'), 'replacement'); + await expect(fs.readFile(join(selected, 'a.txt'), {})).rejects.toThrow(/pinned identities/); + expect(observed.open).not.toHaveBeenCalled(); + }, +); it('never repairs a failed construction by later creating the missing workspace', async () => { const missing = join(base, 'missing-workspace'); @@ -573,3 +588,27 @@ it.runIf(process.platform === 'win32')( expect((await fs.readFile(join(grant, '*.txt'), { glob: true })).content).toContain('grant'); }, ); + +// NI: measures the diagnosed mechanism on the runner itself; the counts are evidence, not a pin. +it.runIf(process.platform === 'win32')( + 'native Windows replacement pairs stay distinct as exact identities', + () => { + let collisions = 0; + let negative = 0; + for (let index = 0; index < 200; index += 1) { + const path = join(base, `probe-${index}`); + mkdirSync(path); + const original = statSync(path, { bigint: true }); + // The original stays alive, so its file ID cannot be reused for the replacement. + renameSync(path, `${path}-old`); + mkdirSync(path); + const replacement = statSync(path, { bigint: true }); + expect([replacement.dev, replacement.ino]).not.toEqual([original.dev, original.ino]); + if (Number(replacement.ino) === Number(original.ino)) collisions += 1; + if (original.ino < 0n || replacement.ino < 0n) negative += 1; + } + console.log( + `NI probe: ${collisions} of 200 replacement pairs collide as Numbers; ${negative} read negative`, + ); + }, +); From ed690d04b465902bdaa08d0d3c1953756b0a2d51 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 01:37:14 +0300 Subject: [PATCH 56/97] fix(db): read and compare media-store identities as exact bigints The media store compared every live identity as JavaScript numbers: the root pin, file snapshots and the owned-file check that guards deletion, the read's descriptor stats, the publish handle and the temporary sweep's claim directory, child, marker and moved-blob checks. nativeFile read bigint but compared through Number(). A 64-bit file ID exceeds a Number's 53 bits, so a replacement could compare equal. Each identity read now takes { bigint: true } and is projected once into an internal ExactStats record: bigint dev/ino, Number nlink and size, and mtimeMs from mtimeNs by Node's own number formula, so it is bit-identical to the previous value. The claim token maps a value through BigInt.asUintN(64): Node fills bigint stats from a signed array, so a file ID with its top bit set read negative and such a temporary was retained forever. Every value without its top bit set keeps its byte-identical token. A new witness file substitutes every identity with 2^60 + k and proves a real replacement is refused at the root pin (canonical and entry reads), at publication, at the claim directory, before a claim begins and at the read's second descriptor stat. The exact-claim vectors gain two negative identities. CAS_COOPERATIVE_READ's anchor follows the new parameter type. Refs: ADR-0106, NI Co-Authored-By: Claude Opus 5.5 --- .../src/media-store-native-identity.test.ts | 339 ++++++++++++++++++ packages/db/src/media-store-paths.ts | 71 +++- packages/db/src/media-store-read.ts | 18 +- .../db/src/media-store-temporaries.test.ts | 3 + packages/db/src/media-store-temporaries.ts | 80 +++-- packages/db/src/media-store.ts | 19 +- tools/fs-policy-controls/mutations.mjs | 4 +- 7 files changed, 475 insertions(+), 59 deletions(-) create mode 100644 packages/db/src/media-store-native-identity.test.ts diff --git a/packages/db/src/media-store-native-identity.test.ts b/packages/db/src/media-store-native-identity.test.ts new file mode 100644 index 00000000..876ed9e3 --- /dev/null +++ b/packages/db/src/media-store-native-identity.test.ts @@ -0,0 +1,339 @@ +import { createHash, randomUUID } from 'node:crypto'; +import { + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + realpathSync, + renameSync, + rmSync, + statSync, + symlinkSync, + utimesSync, + writeFileSync, + type BigIntStats, + type StatOptions, +} from 'node:fs'; +import type { FileHandle } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { basename, join } from 'node:path'; + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { FilesystemMediaStore } from './media-store.js'; +import { MediaStoreFilesystemError, MediaStorePaths } from './media-store-paths.js'; +import { scanMediaFile } from './media-store-read.js'; + +interface IdentityFixture { + readonly fakes: Map; + readonly mkdirs: string[]; + beforeLstat: ((path: string) => void) | undefined; + afterOpen: ((path: string, file: FileHandle) => void) | undefined; + substitute: (info: unknown, exact: BigIntStats | undefined) => void; +} + +/** + * POSIX inodes stay far below 2^53, so these cases substitute every identity the store reads: each real + * (dev, ino) becomes 2^60 + k. Distinct substitutes stay distinct as bigints but share one Number (the + * spacing there is 256), so a read without `{ bigint: true }` or a comparison through Number lets a real + * replacement compare equal, as a 64-bit Windows file ID can. + */ +const fixture = vi.hoisted((): IdentityFixture => { + const fakes = new Map(); + return { + fakes, + mkdirs: [], + beforeLstat: undefined, + afterOpen: undefined, + substitute: (info, exact) => { + if (exact === undefined || typeof info !== 'object' || info === null || !('ino' in info)) + return; + const key = `${exact.dev}:${exact.ino}`; + const value = fakes.get(key) ?? (1n << 60n) + BigInt(fakes.size); + fakes.set(key, value); + Object.defineProperty(info, 'ino', { + value: typeof info.ino === 'bigint' ? value : Number(value), + }); + }, + }; +}); +vi.mock('node:fs', async (original) => { + const native = await original(); + return { + ...native, + lstatSync: (...args: Parameters) => { + fixture.beforeLstat?.(String(args[0])); + const info = native.lstatSync(...args); + fixture.substitute(info, native.lstatSync(args[0], { bigint: true, throwIfNoEntry: false })); + return info; + }, + statSync: (...args: Parameters) => { + const info = native.statSync(...args); + fixture.substitute(info, native.statSync(args[0], { bigint: true, throwIfNoEntry: false })); + return info; + }, + }; +}); +vi.mock('node:fs/promises', async (original) => { + const native = await original(); + return { + ...native, + open: async (...args: Parameters) => { + const file = await native.open(...args); + const stat = file.stat.bind(file); + Object.defineProperty(file, 'stat', { + configurable: true, + value: async (options?: StatOptions) => { + const info = await stat(options); + fixture.substitute(info, await stat({ bigint: true })); + return info; + }, + }); + fixture.afterOpen?.(String(args[0]), file); + return file; + }, + mkdir: (...args: Parameters) => { + fixture.mkdirs.push(String(args[0])); + return native.mkdir(...args); + }, + }; +}); + +const bytes = Buffer.from('exact native identity media fixtures'); +const digest = createHash('sha256').update(bytes).digest('hex'); +let base: string; +let root: string; +beforeEach(() => { + fixture.fakes.clear(); + fixture.mkdirs.length = 0; + base = realpathSync.native(mkdtempSync(join(tmpdir(), 'relavium-media-native-identity-'))); + root = join(base, 'store'); +}); +afterEach(() => { + fixture.beforeLstat = undefined; + fixture.afterOpen = undefined; + vi.restoreAllMocks(); + rmSync(base, { recursive: true, force: true }); +}); + +/** The fixture must make the original and its replacement collide as Numbers and only as Numbers. */ +function expectNumberCollision(original: bigint, replacement: bigint): void { + expect(replacement).not.toBe(original); + expect(Number(replacement)).toBe(Number(original)); +} + +function refusal(action: () => unknown): string | undefined { + try { + action(); + return undefined; + } catch (error) { + if (error instanceof MediaStoreFilesystemError) return error.reason; + throw error; + } +} + +async function outcome(action: Promise): Promise { + return action.then( + () => 'completed', + (error: unknown) => (error instanceof MediaStoreFilesystemError ? error.reason : 'other'), + ); +} + +/** An old producer temporary whose owner the sweep sees as gone. */ +function abandonedTemporary(): string { + mkdirSync(root, { mode: 0o700 }); + const source = join(root, `media-v1.${process.pid}.${randomUUID()}.tmp`); + writeFileSync(source, new Uint8Array([5, 6]), { mode: 0o600 }); + age(source); + return source; +} +function age(path: string): void { + const old = (Date.now() - 2 * 24 * 60 * 60 * 1000) / 1000; + utimesSync(path, old, old); +} +function absentOwners(onCheck: () => void = () => undefined): void { + vi.spyOn(process, 'kill').mockImplementation(() => { + onCheck(); + throw Object.assign(new Error('fixture absent owner'), { code: 'ESRCH' }); + }); +} + +describe('the media store root pin compares exact native identities', () => { + it('refuses a replaced root directory whose identity collides as a Number', () => { + const paths = new MediaStorePaths(root); + expect(paths.root(true)).toBe(root); + const original = statSync(root, { bigint: true }).ino; + // Keep the original alive so the platform cannot recycle its inode for the replacement. + renameSync(root, `${root}-old`); + mkdirSync(root); + expectNumberCollision(original, statSync(root, { bigint: true }).ino); + expect( + refusal(() => paths.root(false)), + 'CAS_ROOT_PIN_PRECISION', + ).toBe('unsafe_path'); + }); + + it('refuses an unchanged root alias whose target was replaced (the canonical identity)', () => { + const alias = join(base, 'alias'); + mkdirSync(root); + symlinkSync(root, alias, 'junction'); + const paths = new MediaStorePaths(alias); + expect(paths.root(false)).toBe(root); + const original = statSync(root, { bigint: true }).ino; + const entry = lstatSync(alias, { bigint: true }).ino; + renameSync(root, `${root}-old`); + mkdirSync(root); + expectNumberCollision(original, statSync(root, { bigint: true }).ino); + expect(lstatSync(alias, { bigint: true }).ino).toBe(entry); + expect( + refusal(() => paths.root(false)), + 'CAS_ROOT_PIN_PRECISION', + ).toBe('unsafe_path'); + }); + + it('refuses a replaced root alias whose target is unchanged (the entry identity)', () => { + const alias = join(base, 'alias'); + mkdirSync(root); + symlinkSync(root, alias, 'junction'); + const paths = new MediaStorePaths(alias); + expect(paths.root(false)).toBe(root); + const canonical = statSync(root, { bigint: true }).ino; + const original = lstatSync(alias, { bigint: true }).ino; + renameSync(alias, `${alias}-old`); + symlinkSync(root, alias, 'junction'); + expectNumberCollision(original, lstatSync(alias, { bigint: true }).ino); + expect(statSync(root, { bigint: true }).ino).toBe(canonical); + expect( + refusal(() => paths.root(false)), + 'CAS_ROOT_PIN_PRECISION', + ).toBe('unsafe_path'); + }); +}); + +describe('media ownership compares exact native identities', () => { + it('refuses to publish a producer temporary replaced after its exclusive open', async () => { + const store = new FilesystemMediaStore(root); + const moved = join(base, 'producer-original'); + let temp: string | undefined; + let identities: { original: bigint; replacement: bigint } | undefined; + fixture.afterOpen = (path) => { + if (!basename(path).startsWith('media-v1.')) return; + fixture.afterOpen = undefined; + temp = path; + // The swap lands after the producer's exclusive open and before its first ownership check. + renameSync(path, moved); + writeFileSync(path, 'replacement', { mode: 0o600 }); + identities = { + original: lstatSync(moved, { bigint: true }).ino, + replacement: lstatSync(path, { bigint: true }).ino, + }; + }; + const result = await outcome(store.put(bytes, 'audio/L16')); + if (temp === undefined || identities === undefined) + throw new Error('the producer was not swapped'); + expectNumberCollision(identities.original, identities.replacement); + const left = existsSync(temp) ? readFileSync(temp, 'utf8') : undefined; + const published = existsSync(join(root, digest.slice(0, 2), digest.slice(2))); + expect({ result, left, published }, 'CAS_OWNED_IDENTITY_PRECISION').toEqual({ + result: 'unsafe_path', + left: 'replacement', + published: false, + }); + }); + + it('refuses to claim into a replaced claim directory', async () => { + const source = abandonedTemporary(); + const moved = join(base, 'claim-original'); + let identities: { original: bigint; replacement: bigint } | undefined; + // The sweep's first ownership check of its source runs after the claim directory is pinned. + fixture.beforeLstat = (path) => { + if (path !== source) return; + const name = readdirSync(root).find((entry) => entry.startsWith('media-claim-v2.')); + if (name === undefined) return; + fixture.beforeLstat = undefined; + const dir = join(root, name); + const original = lstatSync(dir, { bigint: true }).ino; + renameSync(dir, moved); + mkdirSync(dir, { mode: 0o700 }); + identities = { original, replacement: lstatSync(dir, { bigint: true }).ino }; + }; + absentOwners(); + const result = await new FilesystemMediaStore(root).sweepTemporaries(); + if (identities === undefined) throw new Error('the claim directory was not swapped'); + expectNumberCollision(identities.original, identities.replacement); + expect( + { result, source: existsSync(source), moved: readdirSync(moved) }, + 'MEDIA_TEMP_SAME_PRECISION', + ).toEqual({ + result: { removed: 0, retained: 1, reasons: { unsafe_entry: 1 } }, + source: true, + moved: [], + }); + }); + + it('refuses a producer temporary replaced before its claim begins', async () => { + const source = abandonedTemporary(); + const moved = join(base, 'producer-original'); + let identities: { original: bigint; replacement: bigint } | undefined; + // The owner check runs after the sweep's snapshot of the temporary and before its claim. + absentOwners(() => { + if (identities !== undefined) return; + renameSync(source, moved); + writeFileSync(source, 'replacement', { mode: 0o600 }); + age(source); + identities = { + original: lstatSync(moved, { bigint: true }).ino, + replacement: lstatSync(source, { bigint: true }).ino, + }; + }); + const result = await new FilesystemMediaStore(root).sweepTemporaries(); + if (identities === undefined) throw new Error('the producer was not swapped'); + expectNumberCollision(identities.original, identities.replacement); + // nativeFile refuses before any claim directory exists; the later owned-file check would not. + expect( + { + result, + claims: fixture.mkdirs.filter((path) => basename(path).startsWith('media-claim-v2.')), + left: readFileSync(source, 'utf8'), + }, + 'MEDIA_TEMP_NATIVE_FILE_PRECISION', + ).toEqual({ + result: { removed: 0, retained: 1, reasons: { unsafe_entry: 1 } }, + claims: [], + left: 'replacement', + }); + }); +}); + +describe('a media read compares its descriptor stats exactly', () => { + it('refuses a second descriptor stat whose identity differs only beyond a Number (no real filesystem does this)', async () => { + const store = new FilesystemMediaStore(root); + expect(await store.put(bytes, 'audio/L16')).toBe(`media://sha256-${digest}`); + const path = join(root, digest.slice(0, 2), digest.slice(2)); + let identities: { first: bigint; second: bigint } | undefined; + fixture.afterOpen = (opened, file) => { + if (opened !== path) return; + fixture.afterOpen = undefined; + const stat = file.stat.bind(file); + let first: bigint | undefined; + Object.defineProperty(file, 'stat', { + value: async (options?: StatOptions) => { + const info = await stat(options); + if (typeof info.ino !== 'bigint') return info; + if (first === undefined) first = info.ino; + else { + Object.defineProperty(info, 'ino', { value: first + 1n }); + identities = { first, second: first + 1n }; + } + return info; + }, + }); + }; + const result = await outcome(scanMediaFile(new MediaStorePaths(root), digest, false)); + if (identities === undefined) throw new Error('the second stat was not reached'); + expectNumberCollision(identities.first, identities.second); + expect(result, 'MEDIA_FD_IDENTITY_PRECISION').toBe('unsafe_path'); + }); +}); diff --git a/packages/db/src/media-store-paths.ts b/packages/db/src/media-store-paths.ts index 7fb5c534..040066f6 100644 --- a/packages/db/src/media-store-paths.ts +++ b/packages/db/src/media-store-paths.ts @@ -1,6 +1,13 @@ import { RelaviumError } from '@relavium/shared'; -import { lstatSync, mkdirSync, realpathSync, statSync, type Stats } from 'node:fs'; +import { + lstatSync, + mkdirSync, + realpathSync, + statSync, + type BigIntStats, + type Stats, +} from 'node:fs'; import { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; import { transferStorageFailure, withOwnedFilesystem } from './storage-failure-facts.js'; @@ -27,13 +34,49 @@ function within(root: string, path: string): boolean { interface RootPin { readonly canonical: string; - readonly dev: number; - readonly ino: number; - readonly entryDev: number; - readonly entryIno: number; + readonly dev: bigint; + readonly ino: bigint; + readonly entryDev: bigint; + readonly entryIno: bigint; readonly linked: boolean; } +/** + * One stat with an exact native identity. A 64-bit file ID (NTFS, overlayfs xino) exceeds a Number's + * precision, so `dev`/`ino` stay bigint; `nlink` and `size` are exact as Numbers for every value the + * guards admit, and `mtimeMs` repeats Node's number formula so it keeps its sub-millisecond fraction. + */ +export interface ExactStats { + readonly dev: bigint; + readonly ino: bigint; + readonly nlink: number; + readonly size: number; + readonly mtimeMs: number; + isFile(): boolean; + isDirectory(): boolean; + isSymbolicLink(): boolean; +} + +const NS_PER_SECOND = 1_000_000_000n; + +/** Project one bigint stat; a second stat of a path could observe a different file. */ +export function exactStats(info: BigIntStats): ExactStats { + const file = info.isFile(); + const directory = info.isDirectory(); + const link = info.isSymbolicLink(); + const seconds = info.mtimeNs / NS_PER_SECOND - (info.mtimeNs % NS_PER_SECOND < 0n ? 1n : 0n); + return Object.freeze({ + dev: info.dev, + ino: info.ino, + nlink: Number(info.nlink), + size: Number(info.size), + mtimeMs: Number(seconds) * 1000 + Number(info.mtimeNs - seconds * NS_PER_SECOND) / 1e6, + isFile: () => file, + isDirectory: () => directory, + isSymbolicLink: () => link, + }); +} + /** * Pin the root on its first admitted operation. Reads/GC do not provision a never-used store. * Once pinned, absence/replacement/retargeting revokes it; no later operation silently re-pins. @@ -50,18 +93,18 @@ export class MediaStorePaths { root(create: boolean): string | undefined { try { - let entry: Stats; + let entry: BigIntStats; try { - entry = withOwnedFilesystem(() => lstatSync(this.#lexical)); + entry = withOwnedFilesystem(() => lstatSync(this.#lexical, { bigint: true })); } catch (error) { if (!missing(error)) throw error; if (this.#pin !== undefined) throw new MediaStoreFilesystemError('unsafe_path'); if (!create) return undefined; withOwnedFilesystem(() => mkdirSync(this.#lexical, { recursive: true, mode: 0o700 })); - entry = withOwnedFilesystem(() => lstatSync(this.#lexical)); + entry = withOwnedFilesystem(() => lstatSync(this.#lexical, { bigint: true })); } const canonical = withOwnedFilesystem(() => realpathSync(this.#lexical)); - const info = withOwnedFilesystem(() => statSync(canonical)); + const info = withOwnedFilesystem(() => statSync(canonical, { bigint: true })); if (!info.isDirectory()) throw new MediaStoreFilesystemError('unsafe_path'); const current = { canonical, @@ -155,7 +198,7 @@ export class MediaStorePaths { } } - file(path: string): Stats | undefined { + file(path: string): ExactStats | undefined { const info = this.fileSnapshot(path); // fileSnapshot already excludes additional links; ownership also excludes detachment. if (info !== undefined && info.nlink === 0) throw new MediaStoreFilesystemError('unsafe_path'); @@ -167,10 +210,10 @@ export class MediaStorePaths { * between the kernel's lookup and attribute read. Accept that zero-link snapshot only * for non-owning reads/publication checks; temporary ownership and deletion stay strict. */ - fileSnapshot(path: string): Stats | undefined { + fileSnapshot(path: string): ExactStats | undefined { this.assertParent(path); try { - const info = withOwnedFilesystem(() => lstatSync(path)); + const info = exactStats(withOwnedFilesystem(() => lstatSync(path, { bigint: true }))); if (!info.isFile() || info.isSymbolicLink() || (info.nlink !== 0 && info.nlink !== 1)) throw new MediaStoreFilesystemError('unsafe_path'); return info; @@ -180,7 +223,7 @@ export class MediaStorePaths { } } - assertOwnedFile(path: string, opened: Stats): void { + assertOwnedFile(path: string, opened: ExactStats): void { const current = this.file(path); if ( !opened.isFile() || @@ -202,7 +245,7 @@ export class MediaStorePaths { * owner-controlled-parent assumption; it is not a descriptor-relative parent-swap jail. * Temporary-file ownership and destructive operations must not use this read-only check. */ - assertReadableFile(path: string, opened: Stats): void { + assertReadableFile(path: string, opened: ExactStats): void { const current = this.fileSnapshot(path); if ( !opened.isFile() || diff --git a/packages/db/src/media-store-read.ts b/packages/db/src/media-store-read.ts index 4f24e63d..1f7c2604 100644 --- a/packages/db/src/media-store-read.ts +++ b/packages/db/src/media-store-read.ts @@ -10,7 +10,7 @@ import { type DurableMediaMeta, } from '@relavium/shared'; -import { MediaStoreFilesystemError, MediaStorePaths } from './media-store-paths.js'; +import { exactStats, MediaStoreFilesystemError, MediaStorePaths } from './media-store-paths.js'; /** Internal typed cancellation; caller-provided abort reasons never become diagnostics. */ class MediaReadCancelledError extends Error { @@ -66,9 +66,11 @@ export async function scanMediaFile( let cleanupError: MediaStoreFilesystemError | undefined; try { checkAbort(); - const opened = await file.stat().catch((error: unknown) => { - throw paths.error(error); - }); + const opened = exactStats( + await file.stat({ bigint: true }).catch((error: unknown) => { + throw paths.error(error); + }), + ); if (opened.dev !== admitted.dev) throw new MediaStoreFilesystemError('unsafe_path'); paths.assertReadableFile(path, opened); // This check precedes even the fixed read buffer and every whole-body allocation. @@ -118,9 +120,11 @@ export async function scanMediaFile( } total += bytesRead; } - const finished = await file.stat().catch((error: unknown) => { - throw paths.error(error); - }); + const finished = exactStats( + await file.stat({ bigint: true }).catch((error: unknown) => { + throw paths.error(error); + }), + ); checkAbort(); if (finished.dev !== opened.dev || finished.ino !== opened.ino) throw new MediaStoreFilesystemError('unsafe_path'); diff --git a/packages/db/src/media-store-temporaries.test.ts b/packages/db/src/media-store-temporaries.test.ts index 1d01da05..1ff7ec6b 100644 --- a/packages/db/src/media-store-temporaries.test.ts +++ b/packages/db/src/media-store-temporaries.test.ts @@ -238,6 +238,9 @@ async function exactIdentityClaims(): Promise { first + 1n, 11193352636847878000n, (1n << 64n) - 1n, + // Node's signed bigint stats: a 64-bit file ID with its top bit set reads negative (NI). + -1n, + -(1n << 63n), ].entries()) { const label = `exact-${index}`; const claim = await makeClaim(label, value); diff --git a/packages/db/src/media-store-temporaries.ts b/packages/db/src/media-store-temporaries.ts index 2af51f62..ed924a9a 100644 --- a/packages/db/src/media-store-temporaries.ts +++ b/packages/db/src/media-store-temporaries.ts @@ -1,11 +1,16 @@ import { RelaviumError } from '@relavium/shared'; import { randomUUID } from 'node:crypto'; -import { constants, lstatSync, readdirSync, realpathSync, type Stats } from 'node:fs'; +import { constants, lstatSync, readdirSync, realpathSync } from 'node:fs'; import { mkdir, open, rename, rmdir, unlink } from 'node:fs/promises'; import { join } from 'node:path'; -import { MediaStoreFilesystemError, MediaStorePaths } from './media-store-paths.js'; +import { + exactStats, + MediaStoreFilesystemError, + MediaStorePaths, + type ExactStats, +} from './media-store-paths.js'; import { withOwnedFilesystem, withOwnedFilesystemAsync } from './storage-failure-facts.js'; export type MediaTemporaryRetentionReason = @@ -45,6 +50,7 @@ const CLAIM = new RegExp( const LEGACY_ROOT = new RegExp(`^stream\\.${UUID}\\.tmp$`); const LEGACY_SHARD = new RegExp(`^[0-9a-f]{62}\\.${UUID}\\.tmp$`); const MIN_AGE = 24 * 60 * 60 * 1000; +const MIN_NATIVE_IDENTITY = -(1n << 63n); const MAX_NATIVE_IDENTITY = (1n << 64n) - 1n; const MAX_NATIVE_TOKEN = MAX_NATIVE_IDENTITY.toString(36); @@ -53,8 +59,11 @@ interface NativeIdentity { readonly ino: string; } function nativeIdentityToken(value: bigint): string { - if (value < 0n || value > MAX_NATIVE_IDENTITY) throw new MediaStoreFilesystemError('unsafe_path'); - return value.toString(36); + if (value < MIN_NATIVE_IDENTITY || value > MAX_NATIVE_IDENTITY) + throw new MediaStoreFilesystemError('unsafe_path'); + // Node fills bigint stats from a signed array, so a 64-bit file ID with its top bit set reads + // negative; the token is the unsigned value the filesystem reports. + return BigInt.asUintN(64, value).toString(36); } function canonicalNativeToken(field: string): boolean { return ( @@ -80,10 +89,10 @@ function live(pid: string): Liveness { function code(error: unknown): unknown { return typeof error === 'object' && error !== null && 'code' in error ? error.code : undefined; } -function same(a: Stats, b: Stats): boolean { +function same(a: ExactStats, b: ExactStats): boolean { return a.dev === b.dev && a.ino === b.ino; } -function age(info: Stats, now: number): MediaTemporaryRetentionReason | undefined { +function age(info: ExactStats, now: number): MediaTemporaryRetentionReason | undefined { if (!Number.isFinite(info.mtimeMs) || info.mtimeMs > now) return 'future_timestamp'; return now - info.mtimeMs < MIN_AGE ? 'too_young' : undefined; } @@ -203,7 +212,7 @@ export class MediaStoreTemporaries { const marker = join(root, `media-maintenance-v1.${process.pid}.${randomUUID()}`); const identity = await this.createMarker(marker); try { - const candidates: Array<{ path: string; info: Stats }> = []; + const candidates: Array<{ path: string; info: ExactStats }> = []; // The marker is visible before this scan. New producers recheck after exclusive open; // old admitted producers remain visible here until publication/cleanup. for (const name of this.entries(root)) { @@ -262,7 +271,7 @@ export class MediaStoreTemporaries { throw this.paths.error(error); } } - private async createMarker(path: string): Promise { + private async createMarker(path: string): Promise { this.paths.assertParent(path); return this.acquireMarker( path, @@ -273,9 +282,9 @@ export class MediaStoreTemporaries { /** Keep captured fd identity available when close or the post-close ownership check fails. */ private async acquireMarker( path: string, - assertOwned: (identity: Stats) => void, - cleanup: (identity: Stats) => Promise, - ): Promise { + assertOwned: (identity: ExactStats) => void, + cleanup: (identity: ExactStats) => Promise, + ): Promise { const file = await withOwnedFilesystemAsync(() => open( path, @@ -289,14 +298,18 @@ export class MediaStoreTemporaries { ).catch((error: unknown) => { throw this.paths.error(error); }); - let identity: Stats | undefined; + let identity: ExactStats | undefined; let acquired = false; let cleanupError: MediaStoreFilesystemError | undefined; try { try { - identity = await withOwnedFilesystemAsync(() => file.stat()).catch((error: unknown) => { - throw this.paths.error(error); - }); + identity = exactStats( + await withOwnedFilesystemAsync(() => file.stat({ bigint: true })).catch( + (error: unknown) => { + throw this.paths.error(error); + }, + ), + ); } finally { try { await withOwnedFilesystemAsync(() => file.close()); @@ -327,16 +340,16 @@ export class MediaStoreTemporaries { } } } - private async removeOwned(path: string, identity: Stats): Promise { + private async removeOwned(path: string, identity: ExactStats): Promise { this.paths.assertOwnedFile(path, identity); await withOwnedFilesystemAsync(() => unlink(path)).catch((error: unknown) => { throw this.paths.error(error); }); } - private directory(root: string, name: string, identity?: Stats): Stats { + private directory(root: string, name: string, identity?: ExactStats): ExactStats { this.paths.root(false); const path = join(root, name); - const info = withOwnedFilesystem(() => lstatSync(path)); + const info = exactStats(withOwnedFilesystem(() => lstatSync(path, { bigint: true }))); if ( !info.isDirectory() || info.isSymbolicLink() || @@ -347,10 +360,17 @@ export class MediaStoreTemporaries { this.paths.root(false); return info; } - private child(root: string, name: string, identity: Stats, child: string): Stats | undefined { + private child( + root: string, + name: string, + identity: ExactStats, + child: string, + ): ExactStats | undefined { this.directory(root, name, identity); try { - const info = withOwnedFilesystem(() => lstatSync(join(root, name, child))); + const info = exactStats( + withOwnedFilesystem(() => lstatSync(join(root, name, child), { bigint: true })), + ); if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1) throw new MediaStoreFilesystemError('unsafe_path'); return info; @@ -360,14 +380,14 @@ export class MediaStoreTemporaries { } } /** Exact claim identity supplements, rather than replaces, the existing cooperative path floor. */ - private nativeFile(path: string, expected: Stats): NativeIdentity { + private nativeFile(path: string, expected: ExactStats): NativeIdentity { const info = withOwnedFilesystem(() => lstatSync(path, { bigint: true })); if ( !info.isFile() || info.isSymbolicLink() || info.nlink !== 1n || - Number(info.dev) !== expected.dev || - Number(info.ino) !== expected.ino + info.dev !== expected.dev || + info.ino !== expected.ino ) throw new MediaStoreFilesystemError('unsafe_path'); return { dev: nativeIdentityToken(info.dev), ino: nativeIdentityToken(info.ino) }; @@ -375,7 +395,7 @@ export class MediaStoreTemporaries { private async claim( root: string, source: string, - info: Stats, + info: ExactStats, pid: string, nonce: string, ): Promise { @@ -513,9 +533,9 @@ export class MediaStoreTemporaries { private assertRecoveryMarker( root: string, name: string, - identity: Stats, + identity: ExactStats, marker: string, - markerInfo: Stats, + markerInfo: ExactStats, ): void { const own = this.child(root, name, identity, marker); if ( @@ -530,9 +550,9 @@ export class MediaStoreTemporaries { private async removeRecoveryMarker( root: string, name: string, - identity: Stats, + identity: ExactStats, marker: string, - markerInfo: Stats, + markerInfo: ExactStats, ): Promise { this.assertRecoveryMarker(root, name, identity, marker, markerInfo); await withOwnedFilesystemAsync(() => unlink(join(root, name, marker))); @@ -540,9 +560,9 @@ export class MediaStoreTemporaries { private async cleanupRecovery( root: string, name: string, - identity: Stats, + identity: ExactStats, marker: string, - markerInfo: Stats, + markerInfo: ExactStats, ): Promise { await this.removeRecoveryMarker(root, name, identity, marker, markerInfo); try { diff --git a/packages/db/src/media-store.ts b/packages/db/src/media-store.ts index 0a6d7655..4dc71425 100644 --- a/packages/db/src/media-store.ts +++ b/packages/db/src/media-store.ts @@ -1,9 +1,14 @@ import { createHash } from 'node:crypto'; -import { constants, type Stats } from 'node:fs'; +import { constants } from 'node:fs'; import { open, readdir, rename, rm, type FileHandle } from 'node:fs/promises'; import { join } from 'node:path'; -import { MediaStoreFilesystemError, MediaStorePaths } from './media-store-paths.js'; +import { + exactStats, + MediaStoreFilesystemError, + MediaStorePaths, + type ExactStats, +} from './media-store-paths.js'; import { assertMediaNotAborted, scanMediaFile } from './media-store-read.js'; import { MediaStoreTemporaries, @@ -245,15 +250,17 @@ export class FilesystemMediaStore implements MediaStore { ).catch((error: unknown) => { throw this.#paths.error(error); }); - let identity: Stats | undefined; + let identity: ExactStats | undefined; let closed = false; let published = false; let result: string | undefined; let cleanupError: MediaStoreFilesystemError | undefined; try { - identity = await file.stat().catch((error: unknown) => { - throw this.#paths.error(error); - }); + identity = exactStats( + await file.stat({ bigint: true }).catch((error: unknown) => { + throw this.#paths.error(error); + }), + ); this.#paths.assertOwnedFile(tmp, identity); this.#temporaries.assertProducerAllowed(); const handle = await write(file); diff --git a/tools/fs-policy-controls/mutations.mjs b/tools/fs-policy-controls/mutations.mjs index 7210b8d4..10382b07 100644 --- a/tools/fs-policy-controls/mutations.mjs +++ b/tools/fs-policy-controls/mutations.mjs @@ -186,8 +186,8 @@ mutations.push( control( 'CAS_COOPERATIVE_READ', paths, - ' assertReadableFile(path: string, opened: Stats): void {\n const current = this.fileSnapshot(path);', - ' assertReadableFile(path: string, opened: Stats): void {\n this.assertOwnedFile(path, opened);\n const current = this.fileSnapshot(path);', + ' assertReadableFile(path: string, opened: ExactStats): void {\n const current = this.fileSnapshot(path);', + ' assertReadableFile(path: string, opened: ExactStats): void {\n this.assertOwnedFile(path, opened);\n const current = this.fileSnapshot(path);', 'FS_CAUSAL_CAS_COOPERATIVE_READ', [cooperativeCases], 'preserves native', From fda99a8c8017b7ae0cd234c00ded036df3e0143c Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 01:42:04 +0300 Subject: [PATCH 57/97] fix(repo): compare tooling identities as exact bigints Three maintainer tools compared native identities as JavaScript numbers: the overflow capture's destination check, the budget-replay evidence retention's directory identity (the only thing that tells a lookalike replacement from a completed directory before its recursive deletion) and the lifecycle smoke's replacement precondition. All three now read with { bigint: true }. The capture smoke gains a replacement-collision mode and the lifecycle smoke a completed-replacement-collision mode: each substitutes every identity with 2^60 + k (distinct as bigints, one Number), checks the fixture collides as Numbers only and proves the replacement is refused. Reverting either tool to a Number read or comparison fails its smoke. Refs: NI Co-Authored-By: Claude Opus 5.5 --- .../evidence-retention.mjs | 6 ++- .../budget-replay-compat/lifecycle-smoke.mjs | 40 +++++++++++++++++-- tools/overflow-capture/capture.mjs | 5 ++- tools/overflow-capture/check.mjs | 33 ++++++++++++++- 4 files changed, 74 insertions(+), 10 deletions(-) diff --git a/tools/budget-replay-compat/evidence-retention.mjs b/tools/budget-replay-compat/evidence-retention.mjs index 8f3495b3..4e76d1e6 100644 --- a/tools/budget-replay-compat/evidence-retention.mjs +++ b/tools/budget-replay-compat/evidence-retention.mjs @@ -16,9 +16,11 @@ import { basename, dirname, join } from 'node:path'; const kind = 'relavium-budget-replay-evidence-v1'; function directory(path) { - const stat = lstatSync(path); + // Exact identities: a 64-bit file ID (NTFS, overlayfs xino) exceeds a Number's precision, and this + // identity alone tells a lookalike replacement from the completed directory before its deletion. + const stat = lstatSync(path, { bigint: true }); assert.ok(stat.isDirectory() && !stat.isSymbolicLink(), 'replay evidence directory redirected'); - if (process.getuid !== undefined) assert.equal(stat.uid, process.getuid()); + if (process.getuid !== undefined) assert.equal(stat.uid, BigInt(process.getuid())); return { dev: stat.dev, ino: stat.ino }; } function readRecord(path) { diff --git a/tools/budget-replay-compat/lifecycle-smoke.mjs b/tools/budget-replay-compat/lifecycle-smoke.mjs index fa861244..6c6bc39d 100644 --- a/tools/budget-replay-compat/lifecycle-smoke.mjs +++ b/tools/budget-replay-compat/lifecycle-smoke.mjs @@ -279,6 +279,28 @@ async function closedDiagnosticControl(root, environment, mode) { return { mode, pid: child.pid, closedPipe: true, actualCheckPrimaryRetained: true, ...ended }; } +/** + * Substitute every lstat identity with 2^60 + k: distinct as bigints but one Number (the spacing there + * is 256), so a lookalike differs from the completed directory only beyond a Number's precision. + */ +function exactIdentityLstat() { + const lstat = fs.lstatSync; + const fakes = new Map(); + return (path, ...args) => { + const info = lstat(path, ...args); + const exact = lstat(path, { bigint: true, throwIfNoEntry: false }); + if (info !== undefined && exact !== undefined) { + const key = `${exact.dev}:${exact.ino}`; + const value = fakes.get(key) ?? (1n << 60n) + BigInt(fakes.size); + fakes.set(key, value); + Object.defineProperty(info, 'ino', { + value: typeof info.ino === 'bigint' ? value : Number(value), + }); + } + return info; + }; +} + async function retirementIntervention(root, mode) { const fixture = join(root, `retirement-${mode}`); const repository = join(fixture, 'repository'); @@ -299,7 +321,8 @@ async function retirementIntervention(root, mode) { const current = allocateEvidence(repository, temporary); const owner = fs.readFileSync(join(candidate.owned, 'OWNER.json')); const completion = fs.readFileSync(join(candidate.owned, 'completion.json')); - const inode = fs.lstatSync(candidate.owned).ino; + const collision = mode === 'completed-replacement-collision'; + let inode = fs.lstatSync(candidate.owned, { bigint: true }).ino; const read = fs.readFileSync; const readdir = fs.readdirSync; let validated = false; @@ -307,6 +330,7 @@ async function retirementIntervention(root, mode) { let caught; await overrideBuiltins( [ + ...(collision ? [[fs, 'lstatSync', exactIdentityLstat()]] : []), [ fs, 'readdirSync', @@ -336,12 +360,15 @@ async function retirementIntervention(root, mode) { JSON.stringify({ ...JSON.parse(owner.toString('utf8')), repository: fixture }), ); } else { + if (collision) inode = fs.lstatSync(candidate.owned, { bigint: true }).ino; fs.renameSync(candidate.owned, join(fixture, 'saved-completed')); fs.mkdirSync(candidate.owned); fs.writeFileSync(join(candidate.owned, 'OWNER.json'), owner); - if (mode === 'completed-replacement') + if (mode !== 'active-replacement') fs.writeFileSync(join(candidate.owned, 'completion.json'), completion); - assert.notEqual(fs.lstatSync(candidate.owned).ino, inode); + const replaced = fs.lstatSync(candidate.owned, { bigint: true }).ino; + assert.notEqual(replaced, inode); + if (collision) assert.equal(Number(replaced), Number(inode)); } fs.writeFileSync(join(candidate.owned, 'sentinel'), 'preserve unexpected claim'); intervened = true; @@ -382,7 +409,12 @@ export async function checkLifecycleGuards(owned, environment) { }; for (const mode of ['Error', 'undefined', 'null']) results.diagnosticPipes.push(await closedDiagnosticControl(root, environment, mode)); - for (const mode of ['foreign-owner', 'active-replacement', 'completed-replacement']) + for (const mode of [ + 'foreign-owner', + 'active-replacement', + 'completed-replacement', + 'completed-replacement-collision', + ]) results.retirementClaims.push(await retirementIntervention(root, mode)); fs.writeFileSync(join(root, 'results.json'), `${JSON.stringify(results, null, 2)}\n`); return results; diff --git a/tools/overflow-capture/capture.mjs b/tools/overflow-capture/capture.mjs index f5ed5f58..43b9c246 100644 --- a/tools/overflow-capture/capture.mjs +++ b/tools/overflow-capture/capture.mjs @@ -58,7 +58,8 @@ async function main() { process.on('SIGINT', abort); process.on('SIGTERM', abort); try { - identity = fstatSync(fd); + // Exact identities: a 64-bit file ID (NTFS, overlayfs xino) exceeds a Number's precision. + identity = fstatSync(fd, { bigint: true }); const artifact = await captureResponse(options, key, { fetch, newAbortController: () => new globalThis.AbortController(), @@ -70,7 +71,7 @@ async function main() { signal: caller.signal, }); const stillOwnsDestination = () => { - const current = lstatSync(options.out, { throwIfNoEntry: false }); + const current = lstatSync(options.out, { bigint: true, throwIfNoEntry: false }); return current?.dev === identity.dev && current.ino === identity.ino; }; if (!stillOwnsDestination()) throw new CaptureError('destination_changed'); diff --git a/tools/overflow-capture/check.mjs b/tools/overflow-capture/check.mjs index 56d8357c..239f34ba 100644 --- a/tools/overflow-capture/check.mjs +++ b/tools/overflow-capture/check.mjs @@ -44,9 +44,26 @@ import { syncBuiltinESMExports } from 'node:module'; // Delay-only regression: return the real lstat result after the parent swaps the pathname. The fixed // runner has no failure-path lstat/unlink, so it closes before this scheduling point can be reached. const rawLstat = fs.lstatSync; +const rawFstat = fs.fstatSync; +// Exact identity probe: every native identity becomes 2^60 + k, distinct as bigints but one Number. +const fakes = new Map(); +const collision = process.env.CAPTURE_PROBE_MODE === 'replacement-collision'; +function substitute(info, exact) { + if (!collision || info === undefined || exact === undefined) return info; + const key = exact.dev + ':' + exact.ino; + const value = fakes.get(key) ?? (1n << 60n) + BigInt(fakes.size); + fakes.set(key, value); + Object.defineProperty(info, 'ino', { value: typeof info.ino === 'bigint' ? value : Number(value) }); + return info; +} +if (collision) + process.on('exit', () => writeFileSync(process.env.CAPTURE_PROBE_MARKER + '.identities', JSON.stringify([...fakes.values()].map(String)))); +fs.fstatSync = function(fd, ...args) { + return substitute(rawFstat(fd, ...args), rawFstat(fd, { bigint: true })); +}; let scheduled = false; fs.lstatSync = function(path, ...args) { - const result = rawLstat(path, ...args); + const result = substitute(rawLstat(path, ...args), rawLstat(path, { bigint: true, throwIfNoEntry: false })); if (process.env.CAPTURE_PROBE_MODE === 'cleanup-race' && !scheduled && path === process.env.CAPTURE_PROBE_OUT) { scheduled = true; writeFileSync(process.env.CAPTURE_PROBE_RACE_READY, '1'); @@ -69,10 +86,11 @@ globalThis.fetch = async (url, init) => { queueMicrotask(() => process.emit('SIGTERM')); return new Promise(() => {}); case 'replacement-success': + case 'replacement-collision': case 'replacement': unlinkSync(process.env.CAPTURE_PROBE_OUT); writeFileSync(process.env.CAPTURE_PROBE_OUT, 'replacement user file'); - if (process.env.CAPTURE_PROBE_MODE === 'replacement-success') return new Response('{}', {headers:{'content-type':'application/json'}}); + if (process.env.CAPTURE_PROBE_MODE !== 'replacement') return new Response('{}', {headers:{'content-type':'application/json'}}); throw new Error('refused after replacement'); case 'cleanup-race': case 'transport': throw new Error(${JSON.stringify(key)}); @@ -175,6 +193,17 @@ try { assert.equal(replacedSuccess.code, 1, 'success reported a response at an unowned destination'); assert.ok(!replacedSuccess.stdout.includes('capture saved')); assert.equal(readFileSync(out, 'utf8'), 'replacement user file'); + rmSync(out); + // The replacement differs from the reserved file only beyond a Number's precision (NI). + const collided = await run(args, 'replacement-collision'); + const identities = JSON.parse(readFileSync(`${marker}.identities`, 'utf8')).map(BigInt); + assert.ok( + identities.length >= 2 && identities.every((value) => Number(value) === Number(identities[0])), + 'the identity fixture must collide as Numbers only', + ); + assert.equal(collided.code, 1, 'success reported a response at a Number-colliding replacement'); + assert.ok(!collided.stdout.includes('capture saved')); + assert.equal(readFileSync(out, 'utf8'), 'replacement user file'); console.log( '✓ overflow capture command passed offline stdin, file, secrecy and cancellation checks.', ); From 7d628fab49b1a66af96b05e9ebebb8706f855055 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 01:46:03 +0300 Subject: [PATCH 58/97] test(repo): add native-identity precision causal controls Five filesystem causal controls keep NI's exact comparisons live: the policy pins, the catalog open check, the media root pin, the owned-file check and the temporary sweep's same(). Each removal returns that site's whole comparison to Number and must break a real replacement in its complete witness file (four CLI cases or seven media-store cases), whose substituted identities (2^60 + k) differ only beyond a Number's precision. Each control passed its fixed, removal and restored phases through the real runner. The declaration grows to 115 controls / 345 phases; the round-robin shards hold 58 and 57 controls under the unchanged retained-evidence ceilings. Refs: ADR-0105, ADR-0106, NI Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 +- tools/fs-policy-controls/README.md | 13 +++++- tools/fs-policy-controls/mutations.mjs | 58 ++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5fef3a0..89d4d55c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -254,7 +254,7 @@ jobs: name: Wave 2 filesystem causal controls (shard ${{ matrix.shard }} of 2) # ADR-0106/0109 extend the original 26 controls with complete media/lifetime workloads. # The 20-minute job historically reached only 46 of 93 controls; the later budget covered 99. - # The current cohort is 110 controls (2026-10-15); this comment does not certify its execution. + # The current cohort is 115 controls (2026-10-25); this comment does not certify its execution. # Per-phase 90-second/test 15-second deadlines and output/evidence ceilings remain unchanged. # Every phase retains a manifest of the whole source graph (about 200 KB each), so the 330 phase # manifests of one export passed the unchanged 64-MiB retained-evidence bound once Wave 3's PR95 diff --git a/tools/fs-policy-controls/README.md b/tools/fs-policy-controls/README.md index 60b8a41a..422d13ec 100644 --- a/tools/fs-policy-controls/README.md +++ b/tools/fs-policy-controls/README.md @@ -176,7 +176,7 @@ retain their original source attribution. The generated-result correction adds one object-preflight control: removing both encoded/exact decoded guards must break the capacity +1 and +3 pre-reservation witnesses in the complete 18-case generated-result suite. Request-input and direct-store ceilings remain separate. -The current declaration in `mutations.mjs` contains 110 controls /330 phases, +The declaration before NI contained 110 controls /330 phases, projecting 1,987 retained files (110 × 18 + 7) under the unchanged 2,000-file /64-MiB ceilings. This is capacity arithmetic, not a measured export or completed capture; historical captures retain their own source heads and actual counts. @@ -193,6 +193,17 @@ secrets family, the declaration now runs as two round-robin shards of 55 control the exports measured 25.65 MiB and 25.59 MiB. That is one local measurement, not a CI capture. +NI (2026-10-25) appends five precision controls: `fs-root-identity-precision`, +`catalog-identity-precision`, `cas-root-pin-precision`, `cas-owned-identity-precision` +and `media-temp-same-precision`. Each removal returns one site's whole native identity +comparison to `Number`. Its complete witness file (four CLI cases or seven media-store +cases) substitutes every identity the code reads with 2^60 + k, which stays distinct as a +bigint but collapses to one Number, so a real replacement must refuse with the named +assertion. The declaration is now 115 controls /345 phases. Its round-robin shards hold +58 and 57 controls, projecting 174 and 171 phase manifests (about 34 MiB at roughly +205 KB each) and 1,051 and 1,033 retained files under the unchanged ceilings; this is +capacity arithmetic until an export measures it. + The declaration is not acceptance evidence by itself. The complete matrix still requires a stable committed head, all fixed/removal/restored phases, named semantic witnesses, child source/dependency receipts, byte-exact restoration, diff --git a/tools/fs-policy-controls/mutations.mjs b/tools/fs-policy-controls/mutations.mjs index 10382b07..034e7b1d 100644 --- a/tools/fs-policy-controls/mutations.mjs +++ b/tools/fs-policy-controls/mutations.mjs @@ -1434,3 +1434,61 @@ mutations.push({ }, ], }); + +// NI: every file and directory identity compares exact bigints. Each removal returns one site's whole +// comparison to Number and must break a real replacement whose substituted identities (2^60 + k) +// differ only beyond a Number's precision, as a 64-bit Windows file ID can. +const nativeIdentityCli = 'apps/cli/src/engine/tool-host/native-identity.test.ts'; +const nativeIdentityMedia = 'packages/db/src/media-store-native-identity.test.ts'; +mutations.push( + mediaControl( + 'FS_ROOT_IDENTITY_PRECISION', + policy, + ' current.dev !== pin.dev ||\n current.ino !== pin.ino ||\n current.lexicalDev !== pin.lexicalDev ||\n current.lexicalIno !== pin.lexicalIno ||', + ' Number(current.dev) !== Number(pin.dev) ||\n Number(current.ino) !== Number(pin.ino) ||\n Number(current.lexicalDev) !== Number(pin.lexicalDev) ||\n Number(current.lexicalIno) !== Number(pin.lexicalIno) ||', + 'FS_ROOT_IDENTITY_PRECISION', + nativeIdentityCli, + 'refuses a replaced selected directory whose identity collides as a Number', + 4, + ), + mediaControl( + 'CATALOG_IDENTITY_PRECISION', + 'apps/cli/src/workflows/catalog.ts', + 'opened.dev !== stats.dev || opened.ino !== stats.ino', + 'Number(opened.dev) !== Number(stats.dev) || Number(opened.ino) !== Number(stats.ino)', + 'CATALOG_IDENTITY_PRECISION', + nativeIdentityCli, + 'refuses a catalog file swapped between its admission and its open', + 4, + ), + mediaControl( + 'CAS_ROOT_PIN_PRECISION', + paths, + ' pin.dev !== info.dev ||\n pin.ino !== info.ino ||\n pin.entryDev !== entry.dev ||\n pin.entryIno !== entry.ino ||', + ' Number(pin.dev) !== Number(info.dev) ||\n Number(pin.ino) !== Number(info.ino) ||\n Number(pin.entryDev) !== Number(entry.dev) ||\n Number(pin.entryIno) !== Number(entry.ino) ||', + 'CAS_ROOT_PIN_PRECISION', + nativeIdentityMedia, + 'refuses a replaced root directory whose identity collides as a Number', + 7, + ), + mediaControl( + 'CAS_OWNED_IDENTITY_PRECISION', + paths, + ' current.dev !== opened.dev ||\n current.ino !== opened.ino', + ' Number(current.dev) !== Number(opened.dev) ||\n Number(current.ino) !== Number(opened.ino)', + 'CAS_OWNED_IDENTITY_PRECISION', + nativeIdentityMedia, + 'refuses to publish a producer temporary replaced after its exclusive open', + 7, + ), + mediaControl( + 'MEDIA_TEMP_SAME_PRECISION', + mediaTemp, + 'function same(a: ExactStats, b: ExactStats): boolean {\n return a.dev === b.dev && a.ino === b.ino;\n}', + 'function same(a: ExactStats, b: ExactStats): boolean {\n return Number(a.dev) === Number(b.dev) && Number(a.ino) === Number(b.ino);\n}', + 'MEDIA_TEMP_SAME_PRECISION', + nativeIdentityMedia, + 'refuses to claim into a replaced claim directory', + 7, + ), +); From 209e7193bd947c53d6a2005deb138254286fe375 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 01:49:09 +0300 Subject: [PATCH 59/97] docs(repo): record the NI implementation The built-in tools reference states that the policy pins and every media store identity check compare exact bigint device and inode, and that a 64-bit file ID with its top bit set keeps an unsigned claim token. ADR-0105 and ADR-0106 gain dated correction notes. deferred-tasks.md marks the NI entry implemented and under review and records two scoped limits (ReFS's non-unique 64-bit file IDs and the fail-closed source-graph hard-link check). The phase register records the implementation checkpoint with its 28-mutant sweep, and the status lines name NI as under review. Refs: ADR-0105, ADR-0106, NI Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 +- CLAUDE.md | 4 +- ...invocation-grants-not-project-authority.md | 14 +++++ ...ication-requires-bounded-verified-bytes.md | 16 ++++++ docs/reference/shared-core/built-in-tools.md | 9 ++-- docs/roadmap/current.md | 6 +-- docs/roadmap/deferred-tasks.md | 22 +++++++- .../phase-2.5.5-hardening-and-remediation.md | 53 +++++++++++++++++++ 8 files changed, 117 insertions(+), 11 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 5e88e8c5..7753d294 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,7 +117,7 @@ explicit; source CI does not certify later documentation commits. The joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now -accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI and Steps 9b–19 have not started) and whole-wave acceptance remains Step 19. +accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI is implemented and under review, and Steps 9b–19 have not started) and whole-wave acceptance remains Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification @@ -130,7 +130,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a Windows file-identity security fix the maintainer approved on 2026-10-24, register row NI) is next, before Step 9b. `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-implementation-checkpoint--2026-10-25)): every file and directory identity check now compares exact bigints. `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. See [docs/roadmap/current.md](docs/roadmap/current.md) for live status. diff --git a/CLAUDE.md b/CLAUDE.md index 92cf4612..7f225dd2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -206,7 +206,7 @@ explicit; source CI does not certify later documentation commits. The joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now -accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI and Steps 9b–19 have not started) and whole-wave acceptance remains Step 19. +accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI is implemented and under review, and Steps 9b–19 have not started) and whole-wave acceptance remains Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification @@ -219,7 +219,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a Windows file-identity security fix the maintainer approved on 2026-10-24, register row NI) is next, before Step 9b. `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-implementation-checkpoint--2026-10-25)): every file and directory identity check now compares exact bigints. `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. For live status, per-PR history, milestone dates, and open obligations, see the canonical diff --git a/docs/decisions/0105-filesystem-roots-are-invocation-grants-not-project-authority.md b/docs/decisions/0105-filesystem-roots-are-invocation-grants-not-project-authority.md index 4e6d623c..6214e7e5 100644 --- a/docs/decisions/0105-filesystem-roots-are-invocation-grants-not-project-authority.md +++ b/docs/decisions/0105-filesystem-roots-are-invocation-grants-not-project-authority.md @@ -323,3 +323,17 @@ future feature, deferral or claim of merge-commit execution. ## Wave 3 decision acceptance — 2026-10-16 > Amended 2026-10-16: [ADR-0111](0111-cli-meta-operations-use-a-lazy-packaged-runtime.md) is accepted by the maintainer. Meta operations deliberately skip directory existence/link/permission grant admission while retaining pure grammar/count/size checks; no filesystem authority is minted. Execution preserves filesystem admission before the one catalog seed and later model/cost admission. ADR-0112 separately narrows built-in chat HTTP authority without changing filesystem grants. The approved [execution plan](../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) owns implementation and review gates; this note does not claim implementation landing. + +## Exact native identity correction — 2026-10-25 + +The root pins this decision requires compared device and inode as JavaScript numbers. On +Windows `ino` is a 64-bit NTFS file ID, and Linux overlayfs's `xino` feature sets high inode +bits; neither fits a Number's 53 bits, so a replaced root could compare equal (Step 9a.2a's +push CI observed it once on the Windows runner). The policy pins and the workflow catalog's +open check now read and compare exact bigints; the catalog comparison is Windows' only swap +guard, since `O_NOFOLLOW` is undefined there. The decision is unchanged: this restores the +identity the pins already claimed, and permanent causal controls substitute identities that +differ only beyond a Number's precision. ReFS's 64-bit file IDs are not guaranteed unique; +that limit is recorded in [deferred-tasks.md](../roadmap/deferred-tasks.md#native-file-identity-on-windows--approved-increment-2026-10-24). +The [NI checkpoint](../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-implementation-checkpoint--2026-10-25) +owns review and acceptance; this note claims neither. diff --git a/docs/decisions/0106-media-publication-requires-bounded-verified-bytes.md b/docs/decisions/0106-media-publication-requires-bounded-verified-bytes.md index c3d44b2f..72d49dbc 100644 --- a/docs/decisions/0106-media-publication-requires-bounded-verified-bytes.md +++ b/docs/decisions/0106-media-publication-requires-bounded-verified-bytes.md @@ -547,3 +547,19 @@ as PR #94 (`ab08209e`). The [merge checkpoint](../roadmap/phases/phase-2.5.5-har joins final reviewed source `4cdfe011`, review scopes, execution evidence and preserved limits. Earlier pending/unmerged notes are historical; this landing adds no decision, future feature, deferral or claim of merge-commit execution. + +### Exact native identity correction — 2026-10-25 + +The 2026-10-13 correction made the persisted claim token exact but left every live comparison +on numbers: the root pin, file snapshots and the owned-file check that guards deletion, the +read's descriptor stats, the publish handle, and the sweep's claim-directory, child, marker +and moved-blob checks; `nativeFile` read a bigint but compared it through `Number()`. Each now +compares exact bigint device and inode from a single stat projected once, while `nlink`, `size` +and `mtimeMs` keep their number values (`mtimeMs` by Node's own formula). Node fills bigint +stats from a signed array, so a 64-bit file ID with its top bit set read negative and its claim +token refused it, retaining such a temporary forever. The token now encodes the unsigned value; +every other value keeps a byte-identical token, so persisted claims stay compatible. The +[physical storage contract](../reference/shared-core/built-in-tools.md#physical-media-storage-and-limits) +owns the rules. Permanent causal controls and the +[NI checkpoint](../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-implementation-checkpoint--2026-10-25) +own the proof and its gates. The cooperative root/path floor is unchanged. diff --git a/docs/reference/shared-core/built-in-tools.md b/docs/reference/shared-core/built-in-tools.md index e0b6f2d0..6af235a2 100644 --- a/docs/reference/shared-core/built-in-tools.md +++ b/docs/reference/shared-core/built-in-tools.md @@ -276,7 +276,7 @@ A valid host-issued scratch descriptor conditionally adds only its exact subtree ### CLI authority and floors -The CLI owns an immutable, process-local policy with pinned lexical/canonical directory identities, rechecked at each capability operation. Read/list/glob, mention/completion, discovery and write boundaries check lexical provenance and canonical targets; recursive enumeration prunes denied directories **before descent**. An ordinary extra-root grant never overrides confidentiality or protected-write floors, including full tier and auto mode. +The CLI owns an immutable, process-local policy with pinned lexical/canonical directory identities, rechecked at each capability operation. Device and inode compare as exact bigints: a 64-bit file ID (NTFS, or an overlayfs `xino` inode) exceeds a Number's precision. Read/list/glob, mention/completion, discovery and write boundaries check lexical provenance and canonical targets; recursive enumeration prunes denied directories **before descent**. An ordinary extra-root grant never overrides confidentiality or protected-write floors, including full tier and auto mode. Synchronous root admission uses native canonical resolution, matching asynchronous file operations. Windows short-name and long-name spellings must resolve to the same canonical authority; lexical spellings remain separately pinned for provenance and confidentiality checks. Native path membership alone cannot equate two unresolved aliases. @@ -307,7 +307,7 @@ Synthetic tmp/artifact descriptor tests establish this prerequisite only. Automa ### Physical media storage and limits -The filesystem CAS pins its canonical root on the first admitted operation; reads/GC do not create an unused store, and later absence/replacement/retargeting revokes access. Shards/final descendants cannot be symlinks; files must be regular without extra hard links, with exclusive no-follow temporary ownership, one-fd read identity checks, atomic publication and the same physical containment checked for read, delete and GC. CAS has no pnpm exemption. Handle/port semantics live in the [MediaStore reference](llm-provider-seam.md). +The filesystem CAS pins its canonical root on the first admitted operation; reads/GC do not create an unused store, and later absence/replacement/retargeting revokes access. Every root pin, file snapshot, owned-file, claim-directory, marker and read identity check compares device and inode as exact bigints taken from one stat. Shards/final descendants cannot be symlinks; files must be regular without extra hard links, with exclusive no-follow temporary ownership, one-fd read identity checks, atomic publication and the same physical containment checked for read, delete and GC. CAS has no pnpm exemption. Handle/port semantics live in the [MediaStore reference](llm-provider-seam.md). An opened CAS read descriptor may become unlinked when a cooperating publisher atomically republishes the same handle. Reads validate the current root/shard/final-entry floor independently, retain the one opened regular descriptor (at most one remaining link), and verify its bytes against the requested hash before delivery. They do not require the final pathname to retain that descriptor's inode. Temporary ownership and destructive operations retain their stricter identity checks. @@ -349,7 +349,10 @@ the fixed primary acquisition error or expose native diagnostics. New `media-claim-v2` records encode native bigint device/inode identities as bounded canonical unsigned 64-bit tokens. Claim creation, post-move admission and final recovery -revalidation compare these exact identities without a Number conversion. Existing v1 +revalidation compare these exact identities without a Number conversion. Node reports a +64-bit file ID with its top bit set as a negative bigint; its token is the unsigned value +(`BigInt.asUintN(64, …)`), so such a temporary can be claimed, and every other value keeps +its token. Existing v1 claims retain their safe-integer/canonical validation; an ambiguous unsafe historical token is retained, never promoted to deletion authority. Producer, maintenance and recovery-marker layouts are unchanged. This targeted identity check supplements the diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index f7d48f61..85b1897b 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -138,7 +138,7 @@ The [deferred register](deferred-tasks.md) retains its existing limits; closure **Wave 3 Steps 1–8 merged to `main` on 2026-10-18 as [PR #95](https://github.com/HodeTech/Relavium/pull/95), merge commit `6d2a7372` (22:44 UTC). Wave 3 itself is not complete: Steps 9–19 had not started at -that merge (Steps 9a.1, 9a.2a and 9a.2b are now accepted on `development`; the native-identity increment NI and Steps 9b–19 have not started) and whole-wave acceptance +that merge (Steps 9a.1, 9a.2a and 9a.2b are now accepted on `development`, the native-identity increment NI is implemented and under review, and Steps 9b–19 have not started) and whole-wave acceptance remains Step 19.** The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) joins the completed scope, every review attribution, the independent review's dispositions and the publication state observed at merge. It was the maintainer-authorized, scoped publication checkpoint. @@ -172,7 +172,7 @@ bar and round cap, as its medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, by the author alone). Its other findings are recorded in -[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is independently accepted on `development` at `cf97b7f1` (2026-10-24): its round 13 reported no blocker, high or medium finding and its security sitting was clean, and the round's low findings are recorded in deferred-tasks.md ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)); Step 9a.2 is split into 9a.2a and 9a.2b, and Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24) after three rounds, the third with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)); Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24) after two rounds, the second with no blocker, high or medium finding and no regression ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)), and the native-identity increment NI (a Windows file-identity security fix the maintainer approved on 2026-10-24, register row NI) is next, before Step 9b; the substrate's +[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is independently accepted on `development` at `cf97b7f1` (2026-10-24): its round 13 reported no blocker, high or medium finding and its security sitting was clean, and the round's low findings are recorded in deferred-tasks.md ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)); Step 9a.2 is split into 9a.2a and 9a.2b, and Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24) after three rounds, the third with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)); Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24) after two rounds, the second with no blocker, high or medium finding and no regression ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)), and the native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is implemented on `development` and under review ([checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-implementation-checkpoint--2026-10-25)); the substrate's 9a.1–9d increments carry a dedicated archive/crypto security sitting. Steps 10–19 follow the [increment table](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) and end with whole-wave acceptance at Step 19. `development` is pushed after each accepted increment; PR #95 no longer bounds the @@ -1141,7 +1141,7 @@ an ESLint selector rather than a list. ### Wave 3 — Clear the ground -**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Steps 9a.1, 9a.2a and 9a.2b accepted on `development` (2026-10-24), NI and Steps 9b–19 NOT STARTED — 2026-10-24.** +**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Steps 9a.1, 9a.2a and 9a.2b accepted on `development` (2026-10-24), NI implemented and under review, Steps 9b–19 NOT STARTED — 2026-10-25.** ADR-0110–0113, their linked contracts and the nineteen-increment plan were approved on 2026-10-16. [Steps 1–8 were independently accepted](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) on `development`, with whole Step 8 reviewed at `c83c8575`, and published as the scoped checkpoint diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index ff22d861..b7bf85dd 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -2,7 +2,7 @@ > Status: Living -> Last updated: 2026-10-24 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Step 9a.1 is independently accepted on `development` with its recorded low findings below, Step 9a.2a is independently accepted on `development` with its recorded low findings below, Step 9a.2b is independently accepted on `development` with its recorded low findings below, the native-identity increment NI (its approved entry below) and Steps 9b–19 not started; the PR95 increment is closed on `development` under the maintainer's 2026-10-21 round cap). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's edge cases with the maintainer's 2026-10-21 acceptance bar and round cap and the 2026-10-22 rulings. Existing approved deferrals remain. +> Last updated: 2026-10-25 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Step 9a.1 is independently accepted on `development` with its recorded low findings below, Step 9a.2a is independently accepted on `development` with its recorded low findings below, Step 9a.2b is independently accepted on `development` with its recorded low findings below, the native-identity increment NI implemented on `development` and under review (its entry and scoped limits below) and Steps 9b–19 not started; the PR95 increment is closed on `development` under the maintainer's 2026-10-21 round cap). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's edge cases with the maintainer's 2026-10-21 acceptance bar and round cap and the 2026-10-22 rulings. Existing approved deferrals remain. > The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) records completed prerequisites; full M/N lifecycle and other scheduled work remain with their owners. > > Earlier triage: the Phase-2.6 rewrite triaged every open item, the 2026-07-19 full-project @@ -782,6 +782,9 @@ no code (INT-9A2B-R2-1, INT-9A2B-R2-2, CRIT-9A2B-R2-1, the last also amending th Found by Step 9a.2a's push CI (the advisory Windows job of run 40105286543). The maintainer approved a separate increment, after 9a.2b and before 9b (register row NI), rather than a deferral; this entry tracks it until then. +NI is implemented on `development` (2026-10-25) and under review +([checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-implementation-checkpoint--2026-10-25)); the +item stays open until NI's acceptance and the Windows clause below close it. - [ ] **Identity checks compare `ino` as a JavaScript number, which Windows file IDs overflow.** *High; security, Windows.* `statSync`, `lstatSync`, `fstatSync` and `FileHandle.stat` without `{ bigint: true }` return `ino` as a @@ -796,6 +799,23 @@ increment, after 9a.2b and before 9b (register row NI), rather than a deferral; string), with a replacement test per site that fails when the comparison goes back to a number; the Windows job passes the fs-policy test repeatedly. *(high · the five files above)* +Scoped limits recorded with NI (2026-10-25): + +- [ ] **ReFS and Dev Drive file IDs are 128-bit; the 64-bit index Node reads is not guaranteed unique there.** + *Low; Windows ReFS/Dev Drive.* Microsoft documents that the 64-bit identifier "is not guaranteed to be unique on + ReFS". NI makes every comparison exact, so the claim is exact identity on filesystems whose 64-bit file IDs are + unique, such as NTFS, ext4 and APFS; on ReFS two files could still share an index. Minimal example: a workspace + on a Dev Drive whose replaced subdirectory receives a colliding 64-bit index. FAT is out of scope too: its file + IDs can change on rename, which only makes a check refuse (fail closed). **Re-check:** if ReFS or Dev Drive + workspaces become a supported target, read the 128-bit `FILE_ID_INFO` identity, which needs native code or a + dependency and therefore an ADR. *(low · `apps/cli/src/engine/tool-host/fs-policy.ts`, + `packages/db/src/media-store-paths.ts`)* +- [ ] **The causal-control source graph's hard-link check still compares `ino` as a number.** *Low; tooling, + fail-closed.* `tools/usage-coverage-controls/source-graph.mjs` asserts that each copied source file's `ino` + differs from its original's, so a collision can only refuse a copy falsely; four causal families share the + file, and NI left it unchanged rather than perturb them. **Re-check:** switch both reads to `{ bigint: true }` + the next time that file changes. *(low · `tools/usage-coverage-controls/source-graph.mjs`)* + ## Phase 2.6.5 deferrals — `CR-93` and `CR-95` (long half), recorded 2026-09-06; `CR-70`, `CR-94` (remainder) and the in-turn overflow continuation, recorded 2026-09-14; the errored-or-aborted turn residual, the pre-upgrade free-page bytes and the estimator's calibration, recorded 2026-09-18 **Phase closure — 2026-10-11:** W8 merged to `main` in PR #92 and Phase 2.6.5 is complete diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index c3fd02ae..a3b75068 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -9927,3 +9927,56 @@ Engineering defaults recorded with the rulings: - `CAS_COOPERATIVE_READ` is the one anchor that drifts; it is updated in the same commit. - No identity value enters an error, event, log or JSON. - ADR-0105's cooperative-parent assumption is unchanged. + +### Wave 3 NI implementation checkpoint — 2026-10-25 + +**Scope and source.** The native-identity increment (register row NI; ADR-0105, ADR-0106) is implemented on +`development` after the 9a.2b acceptance and the [NI preflight decisions](#wave-3-ni-preflight-decisions--2026-10-25) +(`451223f7`), in six ordered commits and this record: + +1. **CLI fix.** The filesystem policy's canonical and lexical root pins and the workflow catalog's open check read + with `{ bigint: true }` into bigint fields. A witness file has four injected-identity real-replacement cases, and + the fs-invocation precondition reads exactly. +2. **Windows evidence.** A Windows-only probe in `fs-policy.test.ts` makes 200 rename-and-`mkdir` pairs, asserts + each pair is distinct as exact identities and logs how many collide as Numbers or read negative. Its real + replacement tests repeat twenty times. +3. **Media-store fix.** One internal `ExactStats` projection holds bigint `dev` and `ino` and number `nlink`, `size` + and `mtimeMs`; the last comes from `mtimeNs` by Node's own formula and was bit-identical on 307 checked files, + including fractional and negative times. It serves the root pin, file snapshots, the owned-file check, the + read's descriptor stats, the publish handle and the temporary sweep's claim directory, children, marker and + moved blob. `nativeFile` loses its `Number()`, and the claim token encodes the unsigned value + (`BigInt.asUintN(64, …)`). A witness file has seven cases, and the exact-claim vectors gain two negative + identities. `CAS_COOPERATIVE_READ`'s anchor follows the new parameter type. +4. **Tooling fix.** The overflow capture's destination check, the budget-replay evidence retention's directory + identity and its lifecycle-smoke precondition, each with a substituted-identity mode in its smoke. +5. **Permanent controls.** Five filesystem causal controls: `fs-root-identity-precision`, + `catalog-identity-precision`, `cas-root-pin-precision`, `cas-owned-identity-precision` and + `media-temp-same-precision`. Each passed its fixed, removal and restored phases through the real runner. The + declaration grows to 115 controls / 345 phases, in shards of 58 and 57. +6. **Documents.** The policy and physical storage paragraphs of the + [built-in tools reference](../../reference/shared-core/built-in-tools.md#physical-media-storage-and-limits), + dated corrections to ADR-0105 and ADR-0106, the NI entry and two scoped limits in + [deferred-tasks.md](../deferred-tasks.md#native-file-identity-on-windows--approved-increment-2026-10-24), and + the status lines. + +No runtime dependency or ADR is added, and no persisted format changes. + +**Proof.** The witnesses substitute every identity the code reads with 2^60 + k. Adjacent substitutes there are +distinct bigints but one Number. Each witness asserts that its fixture collides as Numbers only, keeps the original +alive so the platform cannot recycle its inode, and asserts its outcome once with a named marker. + +A recorded sweep at the implementation commits applied 28 mutants, and each failed: + +- every comparison returned to a Number, which failed its named witness — the canonical and lexical reads and the + root pin's canonical and entry reads each have their own case; +- every read without its option, which failed its witness and also fails typecheck and the projection; +- the token's signed refusal restored, which failed on the negative vectors; +- each tool's read or comparison reverted, which failed its smoke. + +Number mutants on `dev` alone are equivalent on every supported platform and are not claimed: Windows `dev` is a +32-bit volume serial, and POSIX device numbers are small. The read's second-stat comparison is tested with a faked +second stat; it is a consistency check that no real filesystem fails. By decision, one number comparison remains: +the causal-control source graph's hard-link check, which can only refuse falsely. It is recorded as a scoped limit. + +**Windows evidence.** The advisory Windows job runs the probe and the repeated replacement tests after the push. The +Windows clause closes after three consecutive green advisory runs on pushed heads, with the logged counts recorded. From 6c943db1ad59789dbf7a6f4cb7efb67bb43062a8 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 03:19:55 +0300 Subject: [PATCH 60/97] fix(repo): close the NI internal rigor pass An internal pass at 209e7193 (claims, platform, security, tests and integration lenses with a completeness critic) confirmed one medium finding: nothing pinned that a negative identity's claim token is the unsigned value, so asUintN(63, ...) or an absolute-value encoding passed the whole suite on the deletion-authority path. The exact-claim vectors now assert each claim name's token against a literal, and both mutants fail. Lows corrected: the recovery-marker fixture uses production's unsigned token; the scratch replacement test repeats; the catalog's byte precheck and the claim-directory guard get tests that fail their mutants; the ADR-0106 note names nativeFile's Number cross-check as what refused a top-bit identity before NI; the FAT scoped limit says FAT/exFAT can fail open; the status lines and the controls' comment say every product identity check; the row status, current.md's date, the budget-replay README count, the fs-controls README's measured export and the checkpoint's wording are corrected, and the checkpoint records the pass. Refs: ADR-0105, ADR-0106, NI Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- CLAUDE.md | 2 +- .../src/engine/tool-host/fs-policy.test.ts | 1 + apps/cli/src/workflows/catalog.test.ts | 19 ++++++++++ ...ication-requires-bounded-verified-bytes.md | 10 +++-- docs/roadmap/current.md | 2 +- docs/roadmap/deferred-tasks.md | 20 +++++----- .../phase-2.5.5-hardening-and-remediation.md | 38 ++++++++++++++++--- .../media-store-marker-acquisition.test.ts | 4 +- packages/db/src/media-store-paths.test.ts | 20 +++++++++- .../db/src/media-store-temporaries.test.ts | 24 +++++++----- tools/budget-replay-compat/README.md | 8 ++-- tools/fs-policy-controls/README.md | 7 ++-- tools/fs-policy-controls/mutations.mjs | 2 +- 14 files changed, 119 insertions(+), 40 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7753d294..7e4f5d0d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -130,7 +130,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-implementation-checkpoint--2026-10-25)): every file and directory identity check now compares exact bigints. `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-implementation-checkpoint--2026-10-25)): every product file and directory identity check now compares exact bigints. `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. See [docs/roadmap/current.md](docs/roadmap/current.md) for live status. diff --git a/CLAUDE.md b/CLAUDE.md index 7f225dd2..3a13752e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -219,7 +219,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-implementation-checkpoint--2026-10-25)): every file and directory identity check now compares exact bigints. `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-implementation-checkpoint--2026-10-25)): every product file and directory identity check now compares exact bigints. `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. For live status, per-PR history, milestone dates, and open obligations, see the canonical diff --git a/apps/cli/src/engine/tool-host/fs-policy.test.ts b/apps/cli/src/engine/tool-host/fs-policy.test.ts index 53269284..c553ce37 100644 --- a/apps/cli/src/engine/tool-host/fs-policy.test.ts +++ b/apps/cli/src/engine/tool-host/fs-policy.test.ts @@ -421,6 +421,7 @@ it.each([ describe('exact private host scratch descriptors (no production allocator)', () => { it.each(['tmp', 'artifacts'])( 'permits only the issued %s tree through both jailed factory tiers', + REPLACEMENT_REPEATS, async (kind) => { const home = join(base, 'home'); const scratch = join(home, '.relavium', kind, 'parent'); diff --git a/apps/cli/src/workflows/catalog.test.ts b/apps/cli/src/workflows/catalog.test.ts index 51c8c1d0..37862863 100644 --- a/apps/cli/src/workflows/catalog.test.ts +++ b/apps/cli/src/workflows/catalog.test.ts @@ -11,6 +11,7 @@ import { import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { MAX_SOURCE_CHARS } from '@relavium/core'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { isCliError } from '../process/errors.js'; @@ -195,6 +196,24 @@ describe('discoverCatalog', () => { expect(entries[0]?.valid).toBe(false); }); + it('refuses an oversized file from its admitting stat before reading a byte', () => { + write('workflows', 'big.yaml', 'a'.repeat(MAX_SOURCE_CHARS * 4 + 1)); + const entries = discoverCatalog({ projectConfigDir: configDir, cwd: proj, kind: 'workflows' }); + expect(entries.map((entry) => ({ valid: entry.valid, error: entry.error }))).toEqual([ + { valid: false, error: 'exceeds the size limit' }, + ]); + expect(vi.mocked(readSync)).not.toHaveBeenCalled(); + }); + + it('admits a dense multibyte file over the character cap in bytes but within it in characters', () => { + // Three bytes per character: half the cap in characters is one and a half times it in bytes. + write('workflows', 'dense.yaml', `# ${'€'.repeat(MAX_SOURCE_CHARS / 2)}\n${VALID_WORKFLOW}`); + const entries = discoverCatalog({ projectConfigDir: configDir, cwd: proj, kind: 'workflows' }); + expect(entries.map((entry) => ({ slug: entry.slug, valid: entry.valid }))).toEqual([ + { slug: 'hello', valid: true }, + ]); + }); + it('flags a symlinked .yaml that resolves to a non-regular file (no readFileSync hang)', () => { write('workflows', 'hello.relavium.yaml', VALID_WORKFLOW); // ensures workflows/ exists // A `.yaml` symlinked to a DIRECTORY (statSync follows it → not a regular file). The same guard rejects a diff --git a/docs/decisions/0106-media-publication-requires-bounded-verified-bytes.md b/docs/decisions/0106-media-publication-requires-bounded-verified-bytes.md index 72d49dbc..ae98fc6f 100644 --- a/docs/decisions/0106-media-publication-requires-bounded-verified-bytes.md +++ b/docs/decisions/0106-media-publication-requires-bounded-verified-bytes.md @@ -554,10 +554,12 @@ The 2026-10-13 correction made the persisted claim token exact but left every li on numbers: the root pin, file snapshots and the owned-file check that guards deletion, the read's descriptor stats, the publish handle, and the sweep's claim-directory, child, marker and moved-blob checks; `nativeFile` read a bigint but compared it through `Number()`. Each now -compares exact bigint device and inode from a single stat projected once, while `nlink`, `size` -and `mtimeMs` keep their number values (`mtimeMs` by Node's own formula). Node fills bigint -stats from a signed array, so a 64-bit file ID with its top bit set read negative and its claim -token refused it, retaining such a temporary forever. The token now encodes the unsigned value; +compares exact bigint device and inode, each read from a single stat; the file and directory +checks share one projection, in which `nlink`, `size` and `mtimeMs` keep their number values +(`mtimeMs` by Node's own formula). Node fills bigint stats from a signed array, so a 64-bit file +ID with its top bit set read negative: `nativeFile`'s Number cross-check refused it, and the +claim token would have too, retaining such a temporary forever. The token now encodes the +unsigned value; every other value keeps a byte-identical token, so persisted claims stay compatible. The [physical storage contract](../reference/shared-core/built-in-tools.md#physical-media-storage-and-limits) owns the rules. Permanent causal controls and the diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index 85b1897b..b09632c4 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -1,7 +1,7 @@ # Current state > Status: Living -> Last updated: 2026-10-24 +> Last updated: 2026-10-25 - **Related**: [README.md](README.md), [phases/phase-2.5-cli-consolidation.md](phases/phase-2.5-cli-consolidation.md), [phases/phase-2.5.5-hardening-and-remediation.md](phases/phase-2.5.5-hardening-and-remediation.md), [phases/phase-2-cli.md](phases/phase-2-cli.md), [deferred-tasks.md](deferred-tasks.md), [../project-structure.md](../project-structure.md), [../tech-stack.md](../tech-stack.md) diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index b7bf85dd..66d2b5ce 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -801,15 +801,17 @@ item stays open until NI's acceptance and the Windows clause below close it. Scoped limits recorded with NI (2026-10-25): -- [ ] **ReFS and Dev Drive file IDs are 128-bit; the 64-bit index Node reads is not guaranteed unique there.** - *Low; Windows ReFS/Dev Drive.* Microsoft documents that the 64-bit identifier "is not guaranteed to be unique on - ReFS". NI makes every comparison exact, so the claim is exact identity on filesystems whose 64-bit file IDs are - unique, such as NTFS, ext4 and APFS; on ReFS two files could still share an index. Minimal example: a workspace - on a Dev Drive whose replaced subdirectory receives a colliding 64-bit index. FAT is out of scope too: its file - IDs can change on rename, which only makes a check refuse (fail closed). **Re-check:** if ReFS or Dev Drive - workspaces become a supported target, read the 128-bit `FILE_ID_INFO` identity, which needs native code or a - dependency and therefore an ADR. *(low · `apps/cli/src/engine/tool-host/fs-policy.ts`, - `packages/db/src/media-store-paths.ts`)* +- [ ] **ReFS, Dev Drive, FAT and exFAT do not give unique, stable 64-bit file IDs.** *Low; Windows ReFS/Dev + Drive and FAT/exFAT volumes.* Microsoft documents that the 64-bit identifier "is not guaranteed to be unique on + ReFS", and that a FAT file ID is derived from the position of the file's directory entry, which renaming a file + to a longer name moves. NI makes every comparison exact, so the claim is exact identity on filesystems whose + 64-bit file IDs are unique, such as NTFS, ext4 and APFS. On ReFS two files could share an index; on FAT or exFAT + a replacement created in a freed directory entry could inherit the pinned ID, so a check would admit it (fail + open), whatever the comparison's precision. Minimal example: a pinned directory `selected` on a FAT volume is + renamed to `selected-old` and a new `selected` is created in the freed entry. **Re-check:** if ReFS, Dev Drive, + FAT or exFAT workspaces become a supported target, read the 128-bit `FILE_ID_INFO` identity, pair the identity + with a second discriminator such as creation time, or refuse such volumes; native code or a dependency needs an + ADR. *(low · `apps/cli/src/engine/tool-host/fs-policy.ts`, `packages/db/src/media-store-paths.ts`)* - [ ] **The causal-control source graph's hard-link check still compares `ino` as a number.** *Low; tooling, fail-closed.* `tools/usage-coverage-controls/source-graph.mjs` asserts that each copied source file's `ino` differs from its original's, so a collision can only refuse a copy falsely; four causal families share the diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index a3b75068..07cf41b6 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -3701,7 +3701,7 @@ No credential values or request captures belong in tracked evidence. | 9a.2 | **Recorded split — manifest, symbolication and package-test receipt:** the private map manifest keyed by `dist/.js`, actual packed CLI and engine frame symbolication against the matching source revision, wrong JS/map/source and missing/malformed-upstream negative controls (product copies via `check:map-controls`, also a required uncached CI job), and the atomic `package_test` receipt flip across seal, binding, consumers, aggregate and both workflows. *Split on 2026-10-24 (maintainer-approved; ID coverage #46/ADR-0111 unchanged) into 9a.2a and 9a.2b below, which replace it as increments ([preflight decisions](#wave-3-step-9a2-preflight-decisions--2026-10-24)).* | | 9a.2a | **Recorded split, 2026-10-24 (maintainer-approved) — decoder, private manifest and symbolication:** an in-house strict source-map decoder; strict mapping and range validation of upstream and final maps in the finalizer; the private map manifest (`staging/manifest.json`, keyed by `dist/.js`, with a per-source digest inventory and the commit's tree), sealed after the archive inspection; real packed CLI and `@relavium/core` frame capture through a tooling-only preload during an offline `relavium import`, run on the finalized `dist` that the inspection proves byte-equal to the tarball; symbolication of those frames against the matching source revision (clean: the checkout's Git objects; dirty, local only: the candidate copy), wired into the seal before the receipt is written. The receipt stays `compiled_package_test`. Security sitting. *Independently accepted on `development` at `a92a7602` on 2026-10-24 ([checkpoint](#wave-3-step-9a2a-implementation-checkpoint--2026-10-24), [acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)).* | | 9a.2b | **Recorded split, 2026-10-24 (maintainer-approved) — product map controls and the package-test receipt:** `check:map-controls` over product copies (wrong JS by line and column, wrong map, permuted sources, wrong source, missing, malformed and truncated upstream maps, a shifted valid upstream map, and the package-test producer's post-receipt rebuild refusals that ADR-0111 lists), run in `pnpm run ci` and as a required uncached CI job inside `ci`'s needs (the required contexts stay `ci` and `coverage`); then the atomic `package_test` flip across seal, binding, consumers, aggregate and both workflows, version 1 kept. Security sitting. *Preflight decisions approved on 2026-10-24 ([decisions](#wave-3-step-9a2b-preflight-decisions--2026-10-24)); independently accepted on `development` at `0f73c253` on 2026-10-24 ([checkpoint](#wave-3-step-9a2b-implementation-checkpoint--2026-10-24), [acceptance](#wave-3-step-9a2b-acceptance--2026-10-24)).* | -| NI | **Native file identity on Windows (maintainer-approved 2026-10-24; after 9a.2b, before 9b):** every file and directory identity check compares `dev`/`ino` read with `{ bigint: true }` — the filesystem policy's pinned roots, the workflow catalog's open check and the media store's pins, read check and temporary claims — so a replaced file or directory can never compare equal on Windows, where `ino` is a 64-bit NTFS file ID that a JavaScript number cannot hold exactly; a replacement test per site. Found by 9a.2a's push CI ([acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)); recorded in [deferred-tasks.md](../deferred-tasks.md#native-file-identity-on-windows--approved-increment-2026-10-24). Security sitting. | +| NI | **Native file identity on Windows (maintainer-approved 2026-10-24; after 9a.2b, before 9b):** every file and directory identity check compares `dev`/`ino` read with `{ bigint: true }` — the filesystem policy's pinned roots, the workflow catalog's open check and the media store's pins, read check and temporary claims — so a replaced file or directory can never compare equal on Windows, where `ino` is a 64-bit NTFS file ID that a JavaScript number cannot hold exactly; a replacement test per site. Found by 9a.2a's push CI ([acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)); recorded in [deferred-tasks.md](../deferred-tasks.md#native-file-identity-on-windows--approved-increment-2026-10-24). Security sitting. *Preflight decisions approved on 2026-10-25 ([decisions](#wave-3-ni-preflight-decisions--2026-10-25)); implemented on `development` and under review ([checkpoint](#wave-3-ni-implementation-checkpoint--2026-10-25)).* | | 9b | **Recorded split — recipient custody:** synthetic-only isolated GnuPG controls (public-only home/keyring, pinned primary and encryption-subkey fingerprints, machine-readable refusal of missing, expired, revoked or unusable recipients), encrypt once and reuse the original ciphertext. Explicit crypto/custody security sitting. Fixture proof never claims production recipient or custody configuration. | | 9c | **Recorded split — offline release state controller:** permission-enforcing offline GitHub/npm controller, permissioned prebuild discovery and reservation, and every rerun, partial, conflict and expiry state, with no overwrite, rebuild or re-encryption on uncertain history. | | 9d | **Recorded split — workflow, runbook and whole-Step-9 landing:** least-privilege actual workflow dependency graph, required artifact/platform aggregation and exact-artifact consumers, lost-staging refusal and the release runbook update. The whole-Step-9 cumulative review retains all 9a–9d changes with preservation traces into Step 8, and the dedicated archive/crypto security sitting closes the step. | @@ -9932,7 +9932,7 @@ Engineering defaults recorded with the rulings: **Scope and source.** The native-identity increment (register row NI; ADR-0105, ADR-0106) is implemented on `development` after the 9a.2b acceptance and the [NI preflight decisions](#wave-3-ni-preflight-decisions--2026-10-25) -(`451223f7`), in six ordered commits and this record: +(`451223f7`), in five ordered commits and this record: 1. **CLI fix.** The filesystem policy's canonical and lexical root pins and the workflow catalog's open check read with `{ bigint: true }` into bigint fields. A witness file has four injected-identity real-replacement cases, and @@ -9942,9 +9942,9 @@ Engineering defaults recorded with the rulings: replacement tests repeat twenty times. 3. **Media-store fix.** One internal `ExactStats` projection holds bigint `dev` and `ino` and number `nlink`, `size` and `mtimeMs`; the last comes from `mtimeNs` by Node's own formula and was bit-identical on 307 checked files, - including fractional and negative times. It serves the root pin, file snapshots, the owned-file check, the - read's descriptor stats, the publish handle and the temporary sweep's claim directory, children, marker and - moved blob. `nativeFile` loses its `Number()`, and the claim token encodes the unsigned value + including fractional and negative times. It serves file snapshots, the owned-file check, the read's + descriptor stats, the publish handle and the temporary sweep's claim directory, children, marker and moved + blob; the root pin keeps its own bigint stats. `nativeFile` loses its `Number()`, and the claim token encodes the unsigned value (`BigInt.asUintN(64, …)`). A witness file has seven cases, and the exact-claim vectors gain two negative identities. `CAS_COOPERATIVE_READ`'s anchor follows the new parameter type. 4. **Tooling fix.** The overflow capture's destination check, the budget-replay evidence retention's directory @@ -9980,3 +9980,31 @@ the causal-control source graph's hard-link check, which can only refuse falsely **Windows evidence.** The advisory Windows job runs the probe and the repeated replacement tests after the push. The Windows clause closes after three consecutive green advisory runs on pushed heads, with the logged counts recorded. + +**Author evidence at `209e7193`.** The full `pnpm run ci` passed (exit 0), including all 115 filesystem causal +controls with the five new ones, the product map controls and both changed smokes. + +**Internal rigor pass, 2026-10-25.** Under the rigor-before-review rule, an internal pass at `209e7193` ran claims, +platform, security, tests and integration lenses with a completeness critic, and every finding was adversarially +verified; the detailed record is local. It confirmed one medium finding: nothing pinned that a negative identity's +claim token is the unsigned value. Two encodings that alias a negative identity onto another value's token +(`asUintN(63, …)` and an absolute value) therefore passed the whole suite on the deletion-authority path. Corrected: +the exact-claim vectors now assert each claim name's token against a literal, and both mutants fail on +`MEDIA_TEMP_NATIVE_IDENTITY_TOKEN`. + +Low findings corrected: +- The recovery-marker fixture builds claim names with production's unsigned token. +- The scratch descriptor's real replacement test repeats like the others. +- The catalog's byte precheck gets two tests, each failing its own mutant: one refuses an oversized file before + any read, the other admits a dense multibyte file within the character cap. +- The claim-directory guard gets a test that retains a regular file under a claim name as an unsafe entry. +- The ADR-0106 note now names `nativeFile`'s Number cross-check, not the token, as what refused a top-bit identity + before NI (the message of commit `ed690d04` still says the token). +- The scoped limit now says FAT and exFAT can fail open, since their file IDs derive from directory-entry positions. +- The status lines and the controls' comment now say every *product* identity check, since one tooling check + remains by decision. +- Also corrected: the row's status, current.md's freshness line, the budget-replay README's replacement-control + count, and this checkpoint's commit count and root-pin wording. + +The pass measured the export capacity: one new control's phase manifest is 203,728 bytes, so a 58-control shard's +174 manifests come to 35,448,672 bytes, 52.8 % of the 64-MiB bound. diff --git a/packages/db/src/media-store-marker-acquisition.test.ts b/packages/db/src/media-store-marker-acquisition.test.ts index f9dd7bac..368d7958 100644 --- a/packages/db/src/media-store-marker-acquisition.test.ts +++ b/packages/db/src/media-store-marker-acquisition.test.ts @@ -104,7 +104,9 @@ function claim(): string { const old = new Date(Date.now() - 25 * 60 * 60 * 1000); utimesSync(blob, old, old); const info = statSync(blob, { bigint: true }); - const name = `media-claim-v2.${absentPid}.${randomUUID()}.${info.dev.toString(36)}.${info.ino.toString(36)}.${absentPid}.${randomUUID()}`; + // Production's token: Node reports a top-bit file ID as a negative bigint, encoded unsigned. + const token = (value: bigint): string => BigInt.asUintN(64, value).toString(36); + const name = `media-claim-v2.${absentPid}.${randomUUID()}.${token(info.dev)}.${token(info.ino)}.${absentPid}.${randomUUID()}`; const directory = join(root, name); renameSync(staging, directory); return directory; diff --git a/packages/db/src/media-store-paths.test.ts b/packages/db/src/media-store-paths.test.ts index 3b06059d..8f7a13d2 100644 --- a/packages/db/src/media-store-paths.test.ts +++ b/packages/db/src/media-store-paths.test.ts @@ -1,4 +1,4 @@ -import { createHash } from 'node:crypto'; +import { createHash, randomUUID } from 'node:crypto'; import { existsSync, linkSync, @@ -184,6 +184,24 @@ it('owns put bytes before awaiting and preserves concurrent atomic/idempotent pu expect((await store.listHandles()).map((entry) => entry.handle)).toEqual([handle]); }); +it('retains a regular file under a claim name as an unsafe entry', async () => { + const name = `media-claim-v2.2147483647.${randomUUID()}.1.2.2147483647.${randomUUID()}`; + writeFileSync(join(root, name), 'not a claim directory'); + vi.spyOn(process, 'kill').mockImplementation(() => { + throw Object.assign(new Error('fixture absent owner'), { code: 'ESRCH' }); + }); + try { + expect(await new FilesystemMediaStore(root).sweepTemporaries()).toEqual({ + removed: 0, + retained: 1, + reasons: { unsafe_entry: 1 }, + }); + } finally { + vi.restoreAllMocks(); + } + expect(readFileSync(join(root, name), 'utf8')).toBe('not a claim directory'); +}); + it('preserves never-created read/GC semantics without provisioning a root', async () => { const missing = join(base, 'not-yet-created'); const store = new FilesystemMediaStore(missing); diff --git a/packages/db/src/media-store-temporaries.test.ts b/packages/db/src/media-store-temporaries.test.ts index 1ff7ec6b..038ad89a 100644 --- a/packages/db/src/media-store-temporaries.test.ts +++ b/packages/db/src/media-store-temporaries.test.ts @@ -232,18 +232,22 @@ async function exactIdentityClaims(): Promise { result: { removed: 0, retained: 1, reasons: { identity_changed: 1 } }, exists: true, }); - for (const [index, value] of [ - BigInt(Number.MAX_SAFE_INTEGER), - first, - first + 1n, - 11193352636847878000n, - (1n << 64n) - 1n, - // Node's signed bigint stats: a 64-bit file ID with its top bit set reads negative (NI). - -1n, - -(1n << 63n), - ].entries()) { + // Literal tokens: an encoding that maps a negative identity onto another value's token fails here. + const exact: ReadonlyArray = [ + [BigInt(Number.MAX_SAFE_INTEGER), '2gosa7pa2gv'], + [first, '2gosa7pa2gw'], + [first + 1n, '2gosa7pa2gx'], + [11193352636847878000n, '2d1i8ct3mhd9s'], + [(1n << 64n) - 1n, '3w5e11264sgsf'], + // Node's signed bigint stats: a 64-bit file ID with its top bit set reads negative, and its + // token is the unsigned value the filesystem reports (NI). + [-1n, '3w5e11264sgsf'], + [-(1n << 63n), '1y2p0ij32e8e8'], + ]; + for (const [index, [value, token]] of exact.entries()) { const label = `exact-${index}`; const claim = await makeClaim(label, value); + expect(basename(claim.dir).split('.')[4], 'MEDIA_TEMP_NATIVE_IDENTITY_TOKEN').toBe(token); expect( await new FilesystemMediaStore(join(base, label)).sweepTemporaries(), 'MEDIA_TEMP_NATIVE_IDENTITY_RECOVERY', diff --git a/tools/budget-replay-compat/README.md b/tools/budget-replay-compat/README.md index 364ab4c3..fce05205 100644 --- a/tools/budget-replay-compat/README.md +++ b/tools/budget-replay-compat/README.md @@ -144,11 +144,13 @@ they never modify the extracted predecessor, whose archived files can be read-on ten concurrent completions, future timestamps, active/interrupted/foreign/redirected records, current-owner refusal and direct-child ownership. Seven additional finalization cases, grouped into three results, preserve Error/primitive primary failures and a failed diagnostic sink while surfacing -finalization-only failures. Fourteen additional production-caller controls cover six stage write +finalization-only failures. Fifteen additional production-caller controls cover six stage write faults, ten genuinely failed retention children, a native failed spawn, three real closed-stderr -pipes retaining `Error`/`undefined`/`null`, and three post-scan owner/physical replacement claims. +pipes retaining `Error`/`undefined`/`null`, and four post-scan owner/physical replacement claims. The completed replacement deliberately reuses the same owner/completion bytes and proves that -directory identity is required. These controls execute the actual helpers/callers; they do not +directory identity is required. Its collision variant (NI) substitutes every lstat identity with +2^60 + k, distinct as a bigint but one Number, so only an exact bigint directory identity refuses +the lookalike. These controls execute the actual helpers/callers; they do not mirror their implementations. The ESM control uses the actual production loader with an inert, pinned TypeScript stand-in; it loads no TS source and is separate from the real predecessor/source/dependency acceptance workers. diff --git a/tools/fs-policy-controls/README.md b/tools/fs-policy-controls/README.md index 422d13ec..8885c29b 100644 --- a/tools/fs-policy-controls/README.md +++ b/tools/fs-policy-controls/README.md @@ -200,9 +200,10 @@ comparison to `Number`. Its complete witness file (four CLI cases or seven media cases) substitutes every identity the code reads with 2^60 + k, which stays distinct as a bigint but collapses to one Number, so a real replacement must refuse with the named assertion. The declaration is now 115 controls /345 phases. Its round-robin shards hold -58 and 57 controls, projecting 174 and 171 phase manifests (about 34 MiB at roughly -205 KB each) and 1,051 and 1,033 retained files under the unchanged ceilings; this is -capacity arithmetic until an export measures it. +58 and 57 controls, projecting 174 and 171 phase manifests and 1,051 and 1,033 retained +files under the unchanged ceilings. One new control's export at `209e7193` measured a phase +manifest of 203,728 bytes, so the larger shard's manifests come to 35,448,672 bytes (52.8 % +of the bound); that is one local measurement, not a CI capture. The declaration is not acceptance evidence by itself. The complete matrix still requires a stable committed head, all fixed/removal/restored phases, diff --git a/tools/fs-policy-controls/mutations.mjs b/tools/fs-policy-controls/mutations.mjs index 034e7b1d..55f9be89 100644 --- a/tools/fs-policy-controls/mutations.mjs +++ b/tools/fs-policy-controls/mutations.mjs @@ -1435,7 +1435,7 @@ mutations.push({ ], }); -// NI: every file and directory identity compares exact bigints. Each removal returns one site's whole +// NI: the product's file and directory identities compare exact bigints. Each removal returns one site's whole // comparison to Number and must break a real replacement whose substituted identities (2^60 + k) // differ only beyond a Number's precision, as a 64-bit Windows file ID can. const nativeIdentityCli = 'apps/cli/src/engine/tool-host/native-identity.test.ts'; From fc5af701601fab971f6635fded092cdc44c16a83 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 03:19:55 +0300 Subject: [PATCH 61/97] docs(repo): record 9a.2b's push CI All fifteen required jobs of run 40137915608 at 10993bb8 passed, including the first CI run of the required product map controls job, and the three advisory jobs passed, Windows among them. Refs: ADR-0111 Co-Authored-By: Claude Opus 5.5 --- .../phases/phase-2.5.5-hardening-and-remediation.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 07cf41b6..b683252e 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -9889,6 +9889,13 @@ line names it, and the contract and the interrupted-runner record say that direc public tarball and receipt. The receipt now records `package_test`; the native-identity increment NI is next, then Step 9b. +**Push CI.** `development` was pushed at `10993bb8` (the acceptance commit on top of `0f73c253`), after a full local +`pnpm run ci` at that head passed with all 25 map controls. All fifteen required jobs of +[run 40137915608](https://github.com/HodeTech/Relavium/actions/runs/40137915608) passed, among them the first CI run +of the required product map controls job, the originating pack, the four same-tarball installed-CLI consumers and +their aggregate, lint, typecheck and test, the engine coverage floor and the causal-control families. The three +advisory jobs passed too, including the Windows job whose earlier failure opened NI. + ### Wave 3 NI preflight decisions — 2026-10-25 A read-only preflight (three readers and a synthesis at `10993bb8`; the detailed record and a Turkish decision From c8c0d41be5aac9dbe3624fdd303a37a8569602c8 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 04:43:39 +0300 Subject: [PATCH 62/97] docs(repo): record the NI acceptance Round 1 at fc5af701, a new complete cumulative review of 451223f7..fc5af701 with every reviewer in its own worktree, reported no blocker, high or medium finding and no regression, and its separate security sitting was clean; the full pnpm run ci passed at that head alongside it. Three lows are recorded in deferred-tasks.md; the fourth (the fs-policy-controls README's 110-control shard sentence) and the documentation nits are corrected here, and this commit changes no code. The NI entry stays open until its Windows clause closes after three green advisory Windows runs; Step 9b is next. Refs: ADR-0105, ADR-0106, NI Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 +- CLAUDE.md | 4 +- docs/roadmap/current.md | 6 +-- docs/roadmap/deferred-tasks.md | 34 +++++++++++++-- .../phase-2.5.5-hardening-and-remediation.md | 42 +++++++++++++++++-- tools/budget-replay-compat/README.md | 2 +- tools/fs-policy-controls/README.md | 6 +-- 7 files changed, 79 insertions(+), 19 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7e4f5d0d..92629c5e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,7 +117,7 @@ explicit; source CI does not certify later documentation commits. The joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now -accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI is implemented and under review, and Steps 9b–19 have not started) and whole-wave acceptance remains Step 19. +accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI is accepted too, and Steps 9b–19 have not started) and whole-wave acceptance remains Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification @@ -130,7 +130,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-implementation-checkpoint--2026-10-25)): every product file and directory identity check now compares exact bigints. `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25): round 1 reported no blocker, high or medium finding and no regression, with a clean security sitting; every product file and directory identity check now compares exact bigints, its low findings are recorded in deferred-tasks.md, and its Windows clause closes after three green advisory Windows runs ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)). Step 9b is next. `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. See [docs/roadmap/current.md](docs/roadmap/current.md) for live status. diff --git a/CLAUDE.md b/CLAUDE.md index 3a13752e..a248693f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -206,7 +206,7 @@ explicit; source CI does not certify later documentation commits. The joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now -accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI is implemented and under review, and Steps 9b–19 have not started) and whole-wave acceptance remains Step 19. +accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI is accepted too, and Steps 9b–19 have not started) and whole-wave acceptance remains Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification @@ -219,7 +219,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-implementation-checkpoint--2026-10-25)): every product file and directory identity check now compares exact bigints. `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25): round 1 reported no blocker, high or medium finding and no regression, with a clean security sitting; every product file and directory identity check now compares exact bigints, its low findings are recorded in deferred-tasks.md, and its Windows clause closes after three green advisory Windows runs ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)). Step 9b is next. `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. For live status, per-PR history, milestone dates, and open obligations, see the canonical diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index b09632c4..5b50c869 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -138,7 +138,7 @@ The [deferred register](deferred-tasks.md) retains its existing limits; closure **Wave 3 Steps 1–8 merged to `main` on 2026-10-18 as [PR #95](https://github.com/HodeTech/Relavium/pull/95), merge commit `6d2a7372` (22:44 UTC). Wave 3 itself is not complete: Steps 9–19 had not started at -that merge (Steps 9a.1, 9a.2a and 9a.2b are now accepted on `development`, the native-identity increment NI is implemented and under review, and Steps 9b–19 have not started) and whole-wave acceptance +that merge (Steps 9a.1, 9a.2a and 9a.2b are now accepted on `development`, the native-identity increment NI is accepted too, and Steps 9b–19 have not started) and whole-wave acceptance remains Step 19.** The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) joins the completed scope, every review attribution, the independent review's dispositions and the publication state observed at merge. It was the maintainer-authorized, scoped publication checkpoint. @@ -172,7 +172,7 @@ bar and round cap, as its medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, by the author alone). Its other findings are recorded in -[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is independently accepted on `development` at `cf97b7f1` (2026-10-24): its round 13 reported no blocker, high or medium finding and its security sitting was clean, and the round's low findings are recorded in deferred-tasks.md ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)); Step 9a.2 is split into 9a.2a and 9a.2b, and Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24) after three rounds, the third with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)); Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24) after two rounds, the second with no blocker, high or medium finding and no regression ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)), and the native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is implemented on `development` and under review ([checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-implementation-checkpoint--2026-10-25)); the substrate's +[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is independently accepted on `development` at `cf97b7f1` (2026-10-24): its round 13 reported no blocker, high or medium finding and its security sitting was clean, and the round's low findings are recorded in deferred-tasks.md ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)); Step 9a.2 is split into 9a.2a and 9a.2b, and Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24) after three rounds, the third with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)); Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24) after two rounds, the second with no blocker, high or medium finding and no regression ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)), and the native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25) after one round with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)), its Windows clause open until three green advisory Windows runs; the substrate's 9a.1–9d increments carry a dedicated archive/crypto security sitting. Steps 10–19 follow the [increment table](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) and end with whole-wave acceptance at Step 19. `development` is pushed after each accepted increment; PR #95 no longer bounds the @@ -1141,7 +1141,7 @@ an ESLint selector rather than a list. ### Wave 3 — Clear the ground -**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Steps 9a.1, 9a.2a and 9a.2b accepted on `development` (2026-10-24), NI implemented and under review, Steps 9b–19 NOT STARTED — 2026-10-25.** +**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Steps 9a.1, 9a.2a, 9a.2b and NI accepted on `development` (2026-10-24/25), Steps 9b–19 NOT STARTED — 2026-10-25.** ADR-0110–0113, their linked contracts and the nineteen-increment plan were approved on 2026-10-16. [Steps 1–8 were independently accepted](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) on `development`, with whole Step 8 reviewed at `c83c8575`, and published as the scoped checkpoint diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index 66d2b5ce..577d58c4 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -2,7 +2,7 @@ > Status: Living -> Last updated: 2026-10-25 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Step 9a.1 is independently accepted on `development` with its recorded low findings below, Step 9a.2a is independently accepted on `development` with its recorded low findings below, Step 9a.2b is independently accepted on `development` with its recorded low findings below, the native-identity increment NI implemented on `development` and under review (its entry and scoped limits below) and Steps 9b–19 not started; the PR95 increment is closed on `development` under the maintainer's 2026-10-21 round cap). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's edge cases with the maintainer's 2026-10-21 acceptance bar and round cap and the 2026-10-22 rulings. Existing approved deferrals remain. +> Last updated: 2026-10-25 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Step 9a.1 is independently accepted on `development` with its recorded low findings below, Step 9a.2a is independently accepted on `development` with its recorded low findings below, Step 9a.2b is independently accepted on `development` with its recorded low findings below, the native-identity increment NI independently accepted on `development` with its recorded low findings below (its Windows clause open), and Steps 9b–19 not started; the PR95 increment is closed on `development` under the maintainer's 2026-10-21 round cap). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's edge cases with the maintainer's 2026-10-21 acceptance bar and round cap and the 2026-10-22 rulings. Existing approved deferrals remain. > The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) records completed prerequisites; full M/N lifecycle and other scheduled work remain with their owners. > > Earlier triage: the Phase-2.6 rewrite triaged every open item, the 2026-07-19 full-project @@ -778,13 +778,39 @@ no code (INT-9A2B-R2-1, INT-9A2B-R2-2, CRIT-9A2B-R2-1, the last also amending th producer's owner-only root) until the next build. **Re-check:** with the composition test of the F5 entry, assert `dist` holds no `.map` file after each refused control build. *(low · tools/cli-package/map-controls-core.mjs)* +## Wave 3 NI — recorded low findings (2026-10-25) + +Round 1 of the native-identity increment at `fc5af701` accepted it with these lows +([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)). **Trigger for +every entry:** the next change to the file it names. + +- [ ] **The lifecycle smoke's precondition reads can revert to numbers unnoticed.** *Low; test-side.* *(round 1, + PLAN-NI-R1-1)* Restoring `fs.lstatSync(candidate.owned).ino` at `tools/budget-replay-compat/lifecycle-smoke.mjs` + lines 325 and 369 still passes `pnpm smoke:budget-replay`: in the collision mode a bigint and a Number never + compare strictly equal, and in the other modes the real inodes differ. Production's retention identity is + witnessed separately. **Re-check:** assert both values are bigints before the comparison; reverting either read + must then fail the smoke. *(low · tools/budget-replay-compat/lifecycle-smoke.mjs)* +- [ ] **Nothing tests the projection's `mtimeMs` formula.** *Low.* *(round 1, NI-T-1)* Replacing it in + `packages/db/src/media-store-paths.ts` `exactStats` with whole milliseconds (`Number(info.mtimeNs / 1_000_000n)`) + or whole seconds passes every media-store test. The bit-identical claim rests on an author probe of 307 files, and + a silent change could shift `listHandles()`'s `mtimeMs` and the sweep's age inputs by up to a second. + **Re-check:** a case that sets a fractional and a negative fractional mtime and asserts the projection is + `Object.is`-equal to `lstatSync(path).mtimeMs`; both mutants must fail. *(low · + packages/db/src/media-store-paths.ts)* +- [ ] **The catalog's dense-multibyte case catches only a byte bound below 1.5 times the character cap.** *Low.* + *(round 1, NI-T-2)* `apps/cli/src/workflows/catalog.test.ts` writes three-byte characters to about + 1.5 × `MAX_SOURCE_CHARS` bytes, so `BigInt(MAX_SOURCE_CHARS * 3)`, `* 2` or a `>=` boundary in `catalog.ts`'s + precheck still pass, although a file of four-byte characters just under the character cap would then be refused. + **Re-check:** use four-byte characters to within one character of 4 × `MAX_SOURCE_CHARS` bytes; the 3×, 2× and + `>=` mutants must fail. *(low · apps/cli/src/workflows/catalog.test.ts)* + ## Native file identity on Windows — approved increment (2026-10-24) Found by Step 9a.2a's push CI (the advisory Windows job of run 40105286543). The maintainer approved a separate increment, after 9a.2b and before 9b (register row NI), rather than a deferral; this entry tracks it until then. -NI is implemented on `development` (2026-10-25) and under review -([checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-implementation-checkpoint--2026-10-25)); the -item stays open until NI's acceptance and the Windows clause below close it. +NI is independently accepted on `development` at `fc5af701` (2026-10-25, +[acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)); the item stays +open until the Windows clause below closes it after three consecutive green advisory Windows runs on pushed heads. - [ ] **Identity checks compare `ino` as a JavaScript number, which Windows file IDs overflow.** *High; security, Windows.* `statSync`, `lstatSync`, `fstatSync` and `FileHandle.stat` without `{ bigint: true }` return `ino` as a diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index b683252e..344dc6f2 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -3701,7 +3701,7 @@ No credential values or request captures belong in tracked evidence. | 9a.2 | **Recorded split — manifest, symbolication and package-test receipt:** the private map manifest keyed by `dist/.js`, actual packed CLI and engine frame symbolication against the matching source revision, wrong JS/map/source and missing/malformed-upstream negative controls (product copies via `check:map-controls`, also a required uncached CI job), and the atomic `package_test` receipt flip across seal, binding, consumers, aggregate and both workflows. *Split on 2026-10-24 (maintainer-approved; ID coverage #46/ADR-0111 unchanged) into 9a.2a and 9a.2b below, which replace it as increments ([preflight decisions](#wave-3-step-9a2-preflight-decisions--2026-10-24)).* | | 9a.2a | **Recorded split, 2026-10-24 (maintainer-approved) — decoder, private manifest and symbolication:** an in-house strict source-map decoder; strict mapping and range validation of upstream and final maps in the finalizer; the private map manifest (`staging/manifest.json`, keyed by `dist/.js`, with a per-source digest inventory and the commit's tree), sealed after the archive inspection; real packed CLI and `@relavium/core` frame capture through a tooling-only preload during an offline `relavium import`, run on the finalized `dist` that the inspection proves byte-equal to the tarball; symbolication of those frames against the matching source revision (clean: the checkout's Git objects; dirty, local only: the candidate copy), wired into the seal before the receipt is written. The receipt stays `compiled_package_test`. Security sitting. *Independently accepted on `development` at `a92a7602` on 2026-10-24 ([checkpoint](#wave-3-step-9a2a-implementation-checkpoint--2026-10-24), [acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)).* | | 9a.2b | **Recorded split, 2026-10-24 (maintainer-approved) — product map controls and the package-test receipt:** `check:map-controls` over product copies (wrong JS by line and column, wrong map, permuted sources, wrong source, missing, malformed and truncated upstream maps, a shifted valid upstream map, and the package-test producer's post-receipt rebuild refusals that ADR-0111 lists), run in `pnpm run ci` and as a required uncached CI job inside `ci`'s needs (the required contexts stay `ci` and `coverage`); then the atomic `package_test` flip across seal, binding, consumers, aggregate and both workflows, version 1 kept. Security sitting. *Preflight decisions approved on 2026-10-24 ([decisions](#wave-3-step-9a2b-preflight-decisions--2026-10-24)); independently accepted on `development` at `0f73c253` on 2026-10-24 ([checkpoint](#wave-3-step-9a2b-implementation-checkpoint--2026-10-24), [acceptance](#wave-3-step-9a2b-acceptance--2026-10-24)).* | -| NI | **Native file identity on Windows (maintainer-approved 2026-10-24; after 9a.2b, before 9b):** every file and directory identity check compares `dev`/`ino` read with `{ bigint: true }` — the filesystem policy's pinned roots, the workflow catalog's open check and the media store's pins, read check and temporary claims — so a replaced file or directory can never compare equal on Windows, where `ino` is a 64-bit NTFS file ID that a JavaScript number cannot hold exactly; a replacement test per site. Found by 9a.2a's push CI ([acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)); recorded in [deferred-tasks.md](../deferred-tasks.md#native-file-identity-on-windows--approved-increment-2026-10-24). Security sitting. *Preflight decisions approved on 2026-10-25 ([decisions](#wave-3-ni-preflight-decisions--2026-10-25)); implemented on `development` and under review ([checkpoint](#wave-3-ni-implementation-checkpoint--2026-10-25)).* | +| NI | **Native file identity on Windows (maintainer-approved 2026-10-24; after 9a.2b, before 9b):** every file and directory identity check compares `dev`/`ino` read with `{ bigint: true }` — the filesystem policy's pinned roots, the workflow catalog's open check and the media store's pins, read check and temporary claims — so a replaced file or directory can never compare equal on Windows, where `ino` is a 64-bit NTFS file ID that a JavaScript number cannot hold exactly; a replacement test per site. Found by 9a.2a's push CI ([acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)); recorded in [deferred-tasks.md](../deferred-tasks.md#native-file-identity-on-windows--approved-increment-2026-10-24). Security sitting. *Preflight decisions approved on 2026-10-25 ([decisions](#wave-3-ni-preflight-decisions--2026-10-25)); independently accepted on `development` at `fc5af701` on 2026-10-25 ([checkpoint](#wave-3-ni-implementation-checkpoint--2026-10-25), [acceptance](#wave-3-ni-acceptance--2026-10-25)); its Windows clause closes after three green advisory Windows runs on pushed heads.* | | 9b | **Recorded split — recipient custody:** synthetic-only isolated GnuPG controls (public-only home/keyring, pinned primary and encryption-subkey fingerprints, machine-readable refusal of missing, expired, revoked or unusable recipients), encrypt once and reuse the original ciphertext. Explicit crypto/custody security sitting. Fixture proof never claims production recipient or custody configuration. | | 9c | **Recorded split — offline release state controller:** permission-enforcing offline GitHub/npm controller, permissioned prebuild discovery and reservation, and every rerun, partial, conflict and expiry state, with no overwrite, rebuild or re-encryption on uncertain history. | | 9d | **Recorded split — workflow, runbook and whole-Step-9 landing:** least-privilege actual workflow dependency graph, required artifact/platform aggregation and exact-artifact consumers, lost-staging refusal and the release runbook update. The whole-Step-9 cumulative review retains all 9a–9d changes with preservation traces into Step 8, and the dedicated archive/crypto security sitting closes the step. | @@ -9968,8 +9968,8 @@ Engineering defaults recorded with the rulings: No runtime dependency or ADR is added, and no persisted format changes. -**Proof.** The witnesses substitute every identity the code reads with 2^60 + k. Adjacent substitutes there are -distinct bigints but one Number. Each witness asserts that its fixture collides as Numbers only, keeps the original +**Proof.** The witnesses substitute every inode the code reads, keyed by its real device and inode, with +2^60 + k; device numbers stay real. Adjacent substitutes there are distinct bigints but one Number. Each witness asserts that its fixture collides as Numbers only, keeps the original alive so the platform cannot recycle its inode, and asserts its outcome once with a named marker. A recorded sweep at the implementation commits applied 28 mutants, and each failed: @@ -9978,7 +9978,8 @@ A recorded sweep at the implementation commits applied 28 mutants, and each fail root pin's canonical and entry reads each have their own case; - every read without its option, which failed its witness and also fails typecheck and the projection; - the token's signed refusal restored, which failed on the negative vectors; -- each tool's read or comparison reverted, which failed its smoke. +- each capture and retention read or comparison reverted, which failed its smoke (the lifecycle smoke's + precondition reads are test-side and were not swept; see the acceptance below). Number mutants on `dev` alone are equivalent on every supported platform and are not claimed: Windows `dev` is a 32-bit volume serial, and POSIX device numbers are small. The read's second-stat comparison is tested with a faked @@ -10015,3 +10016,36 @@ Low findings corrected: The pass measured the export capacity: one new control's phase manifest is 203,728 bytes, so a 58-control shard's 174 manifests come to 35,448,672 bytes, 52.8 % of the 64-MiB bound. + +### Wave 3 NI acceptance — 2026-10-25 + +**The native-identity increment NI is independently accepted on `development` at `fc5af701` (2026-10-25)** under +the Acceptance row as amended on 2026-10-23. Round 1 was a NEW independent complete cumulative review of +`451223f7..fc5af701`, with every reviewer in its own worktree detached at that head. It reported no blocker, high or +medium finding and no regression. Its separate security sitting, run at the same head as its own reviewer and report, +was clean. It verified that a replacement fails closed at the policy pins, the catalog open check, the media root pin, +the owned-file check, the temporary sweep's checks and the tooling retention check, and that the unsigned token is a +bijection on Node's reachable identities. It also verified that no identity value reaches an error, event, log, JSON +or IPC. The plan, correctness and integration lenses were clean. The plan reviewer found every commit's anchors valid +and its packages typechecked, with `CAS_COOPERATIVE_READ` moving in the same commit as its target. It also applied the +five control removals, and each failed its named witness. + +**Author evidence at `fc5af701`.** The full `pnpm run ci` ran alongside the round and passed (exit 0). Every causal-control family completed, among them all 115 filesystem controls with the five NI controls. All 25 product map controls and both changed smokes passed. + +The round's verified findings were four lows and four nits: +- Three lows are recorded in + [deferred-tasks.md](../deferred-tasks.md#wave-3-ni--recorded-low-findings-2026-10-25): the lifecycle smoke's + precondition reads can revert unnoticed; nothing tests the projection's `mtimeMs` formula; the catalog's + dense-multibyte case catches only a byte bound below 1.5 times the character cap. +- The fourth low is corrected in the acceptance commit, which changes no code: the fs-policy-controls README's + 110-control shard sentence is now dated. +- Of the nits, the docs' wording now says the witnesses substitute inodes keyed by device and inode, not whole + identities, and the checkpoint's proof bullet no longer claims a lifecycle-smoke sweep. +- The witness files' comments keep their wording: two nits, that devices stay real and that overlayfs inodes can + exceed 2^53. +- The CLI witness stays off the pinned Windows list: one nit. Its proof uses injected identities and does not + depend on the platform. + +The NI entry stays open until its Windows clause closes: three consecutive green advisory Windows runs on pushed +heads, with the probe's logged counts recorded. Step 9b is next. + diff --git a/tools/budget-replay-compat/README.md b/tools/budget-replay-compat/README.md index fce05205..3c1a4178 100644 --- a/tools/budget-replay-compat/README.md +++ b/tools/budget-replay-compat/README.md @@ -148,7 +148,7 @@ finalization-only failures. Fifteen additional production-caller controls cover faults, ten genuinely failed retention children, a native failed spawn, three real closed-stderr pipes retaining `Error`/`undefined`/`null`, and four post-scan owner/physical replacement claims. The completed replacement deliberately reuses the same owner/completion bytes and proves that -directory identity is required. Its collision variant (NI) substitutes every lstat identity with +directory identity is required. Its collision variant (NI) substitutes every lstat inode with 2^60 + k, distinct as a bigint but one Number, so only an exact bigint directory identity refuses the lookalike. These controls execute the actual helpers/callers; they do not mirror their implementations. The ESM control uses the actual production loader diff --git a/tools/fs-policy-controls/README.md b/tools/fs-policy-controls/README.md index 8885c29b..e46f6cc6 100644 --- a/tools/fs-policy-controls/README.md +++ b/tools/fs-policy-controls/README.md @@ -186,7 +186,7 @@ one per phase and about 200 KB each because it lists the whole source graph, and export's raw manifests share the 64-MiB bound. Once Wave 3's PR95 increment grew that graph, a manifest measured 203,421 bytes and the 330 manifests of one export 67,128,930 bytes, 20,066 over the bound, so CI's export refused (2026-10-22). Like the -secrets family, the declaration now runs as two round-robin shards of 55 controls +secrets family, the 110-control declaration then ran as two round-robin shards of 55 controls (`--shard 1/2` and `--shard 2/2`), each exporting its own evidence: 165 manifests (33,564,465 bytes) and 997 retained files (55 × 18 + 7). Both shards ran locally with `RELAVIUM_ACCEPTANCE_EVIDENCE` set at `2a1f7359`'s source; every control passed, and @@ -197,8 +197,8 @@ NI (2026-10-25) appends five precision controls: `fs-root-identity-precision`, `catalog-identity-precision`, `cas-root-pin-precision`, `cas-owned-identity-precision` and `media-temp-same-precision`. Each removal returns one site's whole native identity comparison to `Number`. Its complete witness file (four CLI cases or seven media-store -cases) substitutes every identity the code reads with 2^60 + k, which stays distinct as a -bigint but collapses to one Number, so a real replacement must refuse with the named +cases) substitutes every inode the code reads, keyed by its real device and inode, with +2^60 + k, which stays distinct as a bigint but collapses to one Number, so a real replacement must refuse with the named assertion. The declaration is now 115 controls /345 phases. Its round-robin shards hold 58 and 57 controls, projecting 174 and 171 phase manifests and 1,051 and 1,033 retained files under the unchanged ceilings. One new control's export at `209e7193` measured a phase From 12883df45d8c7f3cc3c670e537909e741f9b0b8e Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 08:34:16 +0300 Subject: [PATCH 63/97] docs(repo): record the Step 9b preflight decisions A read-only preflight at c8c0d41b mapped Step 9b's obligations, the code it meets and GnuPG's mechanics. The maintainer approved each recommendation on 2026-10-25: a library plus synthetic controls with product paths and workflows unchanged; 9b owns the release binding receipt's schema, validator and sealer; --recipient-file with no keyring; SEIPDv1 with MDC and AES-256; an ephemeral round-trip key; a v4 cv25519/RSA>=3072 key policy for every recipient; a staging-relative container; and a split into 9b.1 and 9b.2, so the plan has 32 increments. Refs: ADR-0111 Co-Authored-By: Claude Opus 5.5 --- .../phase-2.5.5-hardening-and-remediation.md | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 344dc6f2..e40533a4 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -3703,6 +3703,8 @@ No credential values or request captures belong in tracked evidence. | 9a.2b | **Recorded split, 2026-10-24 (maintainer-approved) — product map controls and the package-test receipt:** `check:map-controls` over product copies (wrong JS by line and column, wrong map, permuted sources, wrong source, missing, malformed and truncated upstream maps, a shifted valid upstream map, and the package-test producer's post-receipt rebuild refusals that ADR-0111 lists), run in `pnpm run ci` and as a required uncached CI job inside `ci`'s needs (the required contexts stay `ci` and `coverage`); then the atomic `package_test` flip across seal, binding, consumers, aggregate and both workflows, version 1 kept. Security sitting. *Preflight decisions approved on 2026-10-24 ([decisions](#wave-3-step-9a2b-preflight-decisions--2026-10-24)); independently accepted on `development` at `0f73c253` on 2026-10-24 ([checkpoint](#wave-3-step-9a2b-implementation-checkpoint--2026-10-24), [acceptance](#wave-3-step-9a2b-acceptance--2026-10-24)).* | | NI | **Native file identity on Windows (maintainer-approved 2026-10-24; after 9a.2b, before 9b):** every file and directory identity check compares `dev`/`ino` read with `{ bigint: true }` — the filesystem policy's pinned roots, the workflow catalog's open check and the media store's pins, read check and temporary claims — so a replaced file or directory can never compare equal on Windows, where `ino` is a 64-bit NTFS file ID that a JavaScript number cannot hold exactly; a replacement test per site. Found by 9a.2a's push CI ([acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)); recorded in [deferred-tasks.md](../deferred-tasks.md#native-file-identity-on-windows--approved-increment-2026-10-24). Security sitting. *Preflight decisions approved on 2026-10-25 ([decisions](#wave-3-ni-preflight-decisions--2026-10-25)); independently accepted on `development` at `fc5af701` on 2026-10-25 ([checkpoint](#wave-3-ni-implementation-checkpoint--2026-10-25), [acceptance](#wave-3-ni-acceptance--2026-10-25)); its Windows clause closes after three green advisory Windows runs on pushed heads.* | | 9b | **Recorded split — recipient custody:** synthetic-only isolated GnuPG controls (public-only home/keyring, pinned primary and encryption-subkey fingerprints, machine-readable refusal of missing, expired, revoked or unusable recipients), encrypt once and reuse the original ciphertext. Explicit crypto/custody security sitting. Fixture proof never claims production recipient or custody configuration. | +| 9b.1 | **Recorded split, 2026-10-25 (maintainer-approved) — container, isolated GnuPG runner and encrypt-once:** the strict USTAR dialect extracted from the archive reader with its writer promoted; the private container built from the sealed staging in a staging-relative layout, each record's digest and size checked against the private manifest; an isolated public-only GnuPG runner (GnuPG ≥ 2.2.8; `--recipient-file` with no keyring, agent or trustdb); a colon-listing pre-check that refuses missing, expired, revoked, unusable (v4 with cv25519 or RSA ≥ 3072 only) or unexpected recipients and secret material; `encryptOnce` (SEIPDv1 with MDC and AES-256, exactly one `ENC_TO` for the pinned subkey, the exact produced asset, a no-clobber publish, plaintext cleanup on every path); committed synthetic public fixtures, an ephemeral round-trip key, and the GnuPG prerequisite documented. Library and synthetic controls only: product paths and workflows stay unchanged. Crypto/custody security sitting. | +| 9b.2 | **Recorded split, 2026-10-25 (maintainer-approved) — reuse, binding receipt and scope separation:** retained-ciphertext reuse without re-encryption (digest and size verified, zero GnuPG calls); the release binding receipt's schema v1, its validator and a sealer that writes only after the ciphertext exists (9d adds the origin, map-check and `ci`-run binding); the production recipient loader, absent in Wave 3; synthetic and production scope separation. Crypto/custody security sitting. | | 9c | **Recorded split — offline release state controller:** permission-enforcing offline GitHub/npm controller, permissioned prebuild discovery and reservation, and every rerun, partial, conflict and expiry state, with no overwrite, rebuild or re-encryption on uncertain history. | | 9d | **Recorded split — workflow, runbook and whole-Step-9 landing:** least-privilege actual workflow dependency graph, required artifact/platform aggregation and exact-artifact consumers, lost-staging refusal and the release runbook update. The whole-Step-9 cumulative review retains all 9a–9d changes with preservation traces into Step 8, and the dedicated archive/crypto security sitting closes the step. | | 10 | **CLI behaviour/discovery integration:** #45/#264/#265/#268/#13/#17/#18/#310/#309/#19 under ADR-0110, consuming Steps 6–9. Production owner category matrix, real root/subtree JSON help, pricing/chat aliases, non-echoing Commander refusal and signal-safe deletion consent. G15 is refuted and receives real engine/store/log preservation evidence, with no invented token coalescing. Repeat affected packed gates. | @@ -9398,7 +9400,7 @@ plan amendment (row PR95) and the recorded splits; ID coverage is unchanged apar | --- | --- | | Acceptance | Each increment keeps the existing rule: implement, author gates, commit, then NEW independent complete cumulative review rounds until two consecutive rounds are clean after the last material fix. Comment- or documentation-only nits do not reset the count. Security-touching increments add a separate scoped sitting: PR95, 9a.1, 9a.2, 9b, 9c, 9d (whole Step 9 custody), 10a, 10c, 11b, 13c, 14b, 15c, 16a, 16b, 16c (whole Step 16), 18 and 19b (whole wave). PR95 instead follows the maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)). **Amended 2026-10-23 (maintainer, after Step 9a.1's twelfth round):** from Step 9a.1's round 13 on, every increment is accepted after one NEW independent complete cumulative round that reports no blocker, high or medium finding and no regression; that round's low findings are recorded in [deferred-tasks.md](../deferred-tasks.md) with their file, a minimal example and a re-check note rather than fixed in the loop, and nits are optional. A high or medium finding is still corrected and followed by a new round, and the separate security sittings above are unchanged. | | Publication | `development` is pushed after each accepted increment, so its actual CI (including the four same-tarball platform consumers) is observed per increment. The Wave 3 PR follows Step 19. | -| Plan shape | PR95 + 9a.1, 9a.2, 9b, 9c, 9d + 10a–c + 11a–b + 12a–b + 13a–c + 14a–b + 15a–c + 16a–c + 17a–b + 18 + 19a–b (29 increments). Each split is recorded before its step starts. **Amended 2026-10-24 (maintainer):** 9a.2 is split into 9a.2a and 9a.2b, so the plan has 30 increments ([9a.2 preflight decisions](#wave-3-step-9a2-preflight-decisions--2026-10-24)). **Amended 2026-10-24 (maintainer, E6):** the native-identity increment NI is added after 9a.2b, so the plan has 31 increments ([9a.2a acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)). | +| Plan shape | PR95 + 9a.1, 9a.2, 9b, 9c, 9d + 10a–c + 11a–b + 12a–b + 13a–c + 14a–b + 15a–c + 16a–c + 17a–b + 18 + 19a–b (29 increments). Each split is recorded before its step starts. **Amended 2026-10-24 (maintainer):** 9a.2 is split into 9a.2a and 9a.2b, so the plan has 30 increments ([9a.2 preflight decisions](#wave-3-step-9a2-preflight-decisions--2026-10-24)). **Amended 2026-10-24 (maintainer, E6):** the native-identity increment NI is added after 9a.2b, so the plan has 31 increments ([9a.2a acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)). **Amended 2026-10-25 (maintainer, L8):** 9b is split into 9b.1 and 9b.2, so the plan has 32 increments ([9b preflight decisions](#wave-3-step-9b-preflight-decisions--2026-10-25)). | | Reassignments | G54 moves from row 17 to 11a; #311's implementation moves to 10b and its certification stays in 17a; G36/G46 are re-scoped in 10a to the branch that replaces the raw-stack branch and are certified in 17b; promotion of the Steps 6–10 commands.md sections moves from Step 19 to 10c. | | Interim release guard | 9a.1 adds a fail-closed check to the `release.yml` publish job that refuses any non-release candidate. Consequence, acknowledged: no npm publication, including a v0.1.x patch, until 9d and Wave 4b production custody exist. | | PR95 scrubber residuals | Fixed before 9a: the `api-key:Bearer`/`Basic` leak; raw `?`/`#` userinfo masked unless the text before it is host-shaped and an `=` assignment separates it from the authority `@`, in both the LLM leaf and core's twin predicate (core's other documented differences are recorded, not removed); an extended finite scheme list that masks colon-bearing userinfo for the new schemes; the scanner invariants, prefilter and witnesses; the load-sensitive 8 MiB timing case. LOW-1 and LOW-2 become approved deferrals with their recorded triggers. | @@ -10049,3 +10051,36 @@ The round's verified findings were four lows and four nits: The NI entry stays open until its Windows clause closes: three consecutive green advisory Windows runs on pushed heads, with the probe's logged counts recorded. Step 9b is next. +### Wave 3 Step 9b preflight decisions — 2026-10-25 + +A read-only preflight (four readers and a synthesis at `c8c0d41b`) mapped Step 9b's obligations, the code it meets and +GnuPG's mechanics on this machine and the CI runners; the detailed record and a Turkish decision register are local +and Git-ignored. The maintainer approved each recommendation on 2026-10-25. No new ADR or runtime dependency follows: +GnuPG is release tooling (packaging.md's recipient custody), the container is the in-house strict USTAR dialect the +2026-10-19 C1 ruling requires, and SHA-256 comes from `node:crypto`. The 2026-10-19 C2 ruling stands: GnuPG ≥ 2.2.8 is +required by local `pnpm run ci` and the Ubuntu runner, and the controls fail rather than skip. + +| Decision | Ruling | +| --- | --- | +| Placement | A library plus synthetic controls only. `producePackage`, `sealPackage`, the package-test receipt, the interim release guard and both workflows stay unchanged; encryption in the originating job is Step 9d's. SEC-9A2A-2's trigger (sandbox the capture child when 9b designs the custody job) is re-recorded to 9d with a dated note. | +| Release binding receipt | Step 9b defines its schema v1, its validator and a sealer that writes only after the ciphertext exists. Step 9d adds the origin, map-check and `ci`-run binding and replaces the guard. | +| Recipient selection | `--recipient-file` with `--no-keyring`: the file holds exactly one key, whose only usable encryption subkey is the pinned one, so there is no keyring, import, trustdb or agent. The single-usable-subkey rule is also the unexpected-recipient rule; Wave 4b's export becomes `gpg --export '!'`. | +| Ciphertext profile | SEIPDv1 with MDC and AES-256 (`BEGIN_ENCRYPTION 2 9`) is required and an AEAD profile refused, so `gpg -d \| tar -x` with GnuPG ≥ 2.2.8 stays true. A recipient key must not advertise AEAD, a new Wave 4b key clause. | +| Round-trip key | An ephemeral key per run, in an owner-only short-path home whose agent is stopped and whose socket directory is removed afterwards, with a test that no tracked file holds a PGP private-key block. No private key enters the repository or CI. | +| Key policy | Every recipient, fixtures included, must be OpenPGP v4 with a cv25519 or RSA ≥ 3072 encryption subkey. Expiry refuses at `now` with no margin; Wave 4b may add a margin with the real key. | +| Container layout | Staging-relative: `manifest.json`, `finalized.json`, `metafile-esm.json` and `maps/.js.map`. The 2026-10-19 ruling's "repository-relative paths" is read as relative POSIX paths with no host path; a decrypted, extracted archive is a staging directory that the manifest reader and symbolication consume unchanged. | +| Split | 9b.1 (USTAR, container, GnuPG runner, pre-check, fixtures, `encryptOnce`, prerequisite documentation) and 9b.2 (reuse, binding receipt, production loader, scope separation), each with its own round and security sitting; the plan has 32 increments. | + +Engineering defaults recorded with the rulings: +- Every GnuPG call goes through the existing bounded, shell-free `runCommand` in its own process group. +- The call passes `--no-options --no-autostart --no-keyring --no-encrypt-to --no-auto-key-locate --no-auto-key-retrieve --trust-model always --compress-algo none --status-fd 1 --set-filename`. +- It never passes `--yes`, `--hidden-recipient`, `--throw-keyids`, `--faked-system-time`, `--ignore-*` or `--allow-*`. +- Recipients are classified from a colon listing, because `INV_RECP` reasons do not tell the cases apart. +- Success requires one `BEGIN_ENCRYPTION`, one `END_ENCRYPTION`, exit 0 and exactly one `ENC_TO` for the pinned subkey on the ciphertext. +- An existing ciphertext refuses with zero GnuPG calls, and the ciphertext is published with `wx`. +- Plaintext lives in an owner-only work directory that is removed on every path. +- Errors are fixed `PACKED_CLI_CUSTODY_` codes, and GnuPG's stderr never surfaces. +- The tests join `test:acceptance-tools`, so they run in local `pnpm run ci` and the four Ubuntu jobs already inside `ci`'s needs, with no workflow edit. +- Windows has no GnuPG on its image, and the 2.2.8 floor is enforced only by the version check; both are recorded limits. +- Before the first external round, the custody tests run in a local `ubuntu:24.04` container with the distribution's GnuPG, to gather 2.4.4 evidence. + From 4aeee3e1d3540001266a77c1a473e2b2ec28e7d0 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 08:39:28 +0300 Subject: [PATCH 64/97] refactor(repo): extract the strict USTAR dialect into one reader and writer Step 9b.1's private container needs the same strict USTAR dialect the package archive inspector reads (the 2026-10-19 C1 ruling: in-house, round-tripped, no dependency). ustar.mjs now owns it: readUstar validates every header field, the duplicate and case-alias rules, bounds, padding and the exact end, with the caller's path rule, mode rule, entry hook and refusal reporter; writeUstar and ustarEntry write it. The archive inspector delegates with its own refusal codes and order unchanged, and the test-only writer wraps the shared one, so archive.test.mjs and seal.test.mjs pass unchanged. New tests cover the owner-only dialect, every header refusal, the hook order and extraction by the platform tar. Refs: ADR-0111, Step 9b.1 Co-Authored-By: Claude Opus 5.5 --- package.json | 2 +- tools/cli-package/archive.mjs | 142 +++++++----------------- tools/cli-package/ustar-fixture.mjs | 36 ++---- tools/cli-package/ustar.mjs | 164 ++++++++++++++++++++++++++++ tools/cli-package/ustar.test.mjs | 150 +++++++++++++++++++++++++ 5 files changed, 360 insertions(+), 134 deletions(-) create mode 100644 tools/cli-package/ustar.mjs create mode 100644 tools/cli-package/ustar.test.mjs diff --git a/package.json b/package.json index 10ca2fb2..b1b4c8e1 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "smoke:budget-replay": "node tools/budget-replay-compat/check.mjs", "check:wire-mutations": "node tools/wire-conformance/check.mjs", "check:usage-mutations": "node tools/usage-coverage-controls/check.mjs", - "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/check.test.mjs tools/cli-package/paths.test.mjs tools/cli-package/source-map.test.mjs tools/cli-package/manifest.test.mjs tools/cli-package/symbolicate.test.mjs tools/cli-package/map-controls.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", + "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/ustar.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/check.test.mjs tools/cli-package/paths.test.mjs tools/cli-package/source-map.test.mjs tools/cli-package/manifest.test.mjs tools/cli-package/symbolicate.test.mjs tools/cli-package/map-controls.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", "check:fs-mutations": "node tools/fs-policy-controls/check.mjs", "check:secrets-mutations": "node tools/secrets-controls/check.mjs", "smoke:cli-bootstrap": "node tools/cli-bootstrap/check.mjs", diff --git a/tools/cli-package/archive.mjs b/tools/cli-package/archive.mjs index 9cf20999..ee7b8741 100644 --- a/tools/cli-package/archive.mjs +++ b/tools/cli-package/archive.mjs @@ -2,6 +2,7 @@ import { Buffer } from 'node:buffer'; import { createHash } from 'node:crypto'; import { isDeepStrictEqual, TextDecoder } from 'node:util'; import { gunzipSync, inflateRawSync } from 'node:zlib'; +import { readUstar } from './ustar.mjs'; // This inspects the pinned pnpm pack dialect, not arbitrary tar archives. There // is no extraction and no fallback to a more permissive platform tar executable. @@ -23,9 +24,9 @@ export class PackageArchiveError extends Error { } } -/** @param {string} code @returns {never} */ -function refuse(code) { - throw new PackageArchiveError(code); +/** @param {string} code @param {unknown} [cause] @returns {never} */ +function refuse(code, cause) { + throw new PackageArchiveError(code, cause === undefined ? undefined : { cause }); } /** @param {unknown} value @returns {value is Record} */ @@ -78,22 +79,6 @@ function unpack(bytes) { } } -/** @param {Buffer} field */ -function octal(field) { - const text = field.toString('latin1'); - if (!/^[0-7]+[ \0]*$/u.test(text)) refuse('TAR_NUMBER'); - const value = Number.parseInt(text, 8); - if (!Number.isSafeInteger(value)) refuse('TAR_NUMBER'); - return value; -} - -/** @param {Buffer} field */ -function nameField(field) { - const end = field.indexOf(0); - if (end < 1 || field.subarray(end).some((byte) => byte !== 0)) refuse('TAR_NAME'); - return utf8(field.subarray(0, end)); -} - /** @param {string} path */ export function assertPublicPackagePath(path) { if ( @@ -106,43 +91,6 @@ export function assertPublicPackagePath(path) { if (/\/(?:con|prn|aux|nul|com[1-9]|lpt[1-9])\.js$/iu.test(path)) refuse('PUBLIC_PATH'); } -/** @param {Buffer} header */ -function readHeader(header) { - const checksum = octal(header.subarray(148, 156)); - let actual = 0; - for (let index = 0; index < 512; index += 1) { - actual += index >= 148 && index < 156 ? 32 : (header[index] ?? 0); - } - if (checksum !== actual) refuse('TAR_CHECKSUM'); - if (!header.subarray(257, 265).equals(Buffer.from('ustar\0' + '00'))) refuse('TAR_FORMAT'); - if (header[156] !== 48) refuse('TAR_TYPE'); - // No linkname, owner strings, prefix, extension records or reserved bytes. - // A regular USTAR file is the only dialect produced by our current packer. - for (const [start, end] of [ - [157, 257], - [265, 329], - [345, 512], - ]) { - if (header.subarray(start, end).some((byte) => byte !== 0)) refuse('TAR_METADATA'); - } - for (const [start, end] of [ - [108, 116], - [116, 124], - [329, 337], - [337, 345], - ]) { - if (octal(header.subarray(start, end)) !== 0) refuse('TAR_METADATA'); - } - octal(header.subarray(136, 148)); // Validate mtime syntax without requiring a clock value. - const path = nameField(header.subarray(0, 100)); - assertPublicPackagePath(path); - const mode = octal(header.subarray(100, 108)); - if (mode !== (path === 'package/dist/index.js' ? 0o755 : 0o644)) refuse('TAR_MODE'); - const size = octal(header.subarray(124, 136)); - if (size > ARCHIVE_LIMITS.fileBytes) refuse('FILE_SIZE'); - return { path, mode, size }; -} - /** @param {Buffer} bytes @param {Record} expected */ function manifest(bytes, expected) { if (bytes.length > ARCHIVE_LIMITS.manifestBytes) refuse('MANIFEST_SIZE'); @@ -223,59 +171,45 @@ export function inspectPackageArchive(bytes, { expectedFiles, expectedPackage }) if (bytes.byteLength > ARCHIVE_LIMITS.compressedBytes) refuse('GZIP_SIZE'); const compressed = Buffer.from(bytes); const tar = unpack(compressed); - const seen = new Set(); - const portableNames = new Set(); const files = []; - let offset = 0; - let ended = false; /** @type {Record | undefined} */ let packageManifest; - while (offset < tar.length) { - if (tar.length - offset < 512) refuse('TAR_TRUNCATED'); - const header = tar.subarray(offset, offset + 512); - if (header.every((byte) => byte === 0)) { - if (tar.length - offset !== 1024 || tar.subarray(offset).some((byte) => byte !== 0)) { - refuse('TAR_END'); - } - ended = true; - break; - } - if (files.length >= ARCHIVE_LIMITS.files) refuse('FILE_COUNT'); - const { path, mode, size } = readHeader(header); - if (seen.has(path)) refuse('DUPLICATE_FILE'); - seen.add(path); - if (portableNames.has(path.toLowerCase())) refuse('PATH_ALIAS'); - portableNames.add(path.toLowerCase()); - const expected = expectedFiles.get(path); - if (expected === undefined) refuse('EXTRA_FILE'); - const start = offset + 512; - const end = start + size; - const next = start + Math.ceil(size / 512) * 512; - if (next > tar.length) refuse('TAR_TRUNCATED'); - if (tar.subarray(end, next).some((byte) => byte !== 0)) refuse('TAR_PADDING'); - const contents = tar.subarray(start, end); - if (path === 'package/package.json') packageManifest = manifest(contents, expectedPackage); - if (expected instanceof Uint8Array) { - if (path !== 'package/package.json' && !contents.equals(expected)) refuse('FILE_DRIFT'); - } else if (contents.length !== expected.bytes || digest(contents) !== expected.sha256) { - refuse('FILE_DRIFT'); - } - if (path.endsWith('.js')) { - const source = utf8(contents); - if ( - path === 'package/dist/index.js' - ? !source.startsWith('#!/usr/bin/env node\n') - : source.startsWith('#!') + readUstar(tar, { + admitPath: assertPublicPackagePath, + modeFor: (path) => (path === 'package/dist/index.js' ? 0o755 : 0o644), + maxFiles: ARCHIVE_LIMITS.files, + maxFileBytes: ARCHIVE_LIMITS.fileBytes, + refuse, + admitEntry: (path) => { + if (!expectedFiles.has(path)) refuse('EXTRA_FILE'); + }, + onFile: ({ path, mode, size, contents }) => { + const expected = expectedFiles.get(path); + if (path === 'package/package.json') packageManifest = manifest(contents, expectedPackage); + if (expected instanceof Uint8Array) { + if (path !== 'package/package.json' && !contents.equals(expected)) refuse('FILE_DRIFT'); + } else if ( + expected === undefined || + contents.length !== expected.bytes || + digest(contents) !== expected.sha256 ) { - refuse('HASHBANG'); + refuse('FILE_DRIFT'); } - if (/sourceMappingURL\s*=/u.test(source)) refuse('MAP_LINK'); - } - files.push({ path, bytes: size, sha256: digest(contents), mode }); - offset = next; - } - if (!ended) refuse('TAR_END'); - if (seen.size !== expectedFiles.size || !packageManifest) refuse('MISSING_FILE'); + if (path.endsWith('.js')) { + const source = utf8(contents); + if ( + path === 'package/dist/index.js' + ? !source.startsWith('#!/usr/bin/env node\n') + : source.startsWith('#!') + ) { + refuse('HASHBANG'); + } + if (/sourceMappingURL\s*=/u.test(source)) refuse('MAP_LINK'); + } + files.push({ path, bytes: size, sha256: digest(contents), mode }); + }, + }); + if (files.length !== expectedFiles.size || !packageManifest) refuse('MISSING_FILE'); return { sha256: digest(compressed), bytes: compressed.length, diff --git a/tools/cli-package/ustar-fixture.mjs b/tools/cli-package/ustar-fixture.mjs index 9e7974c6..e48d8711 100644 --- a/tools/cli-package/ustar-fixture.mjs +++ b/tools/cli-package/ustar-fixture.mjs @@ -1,42 +1,20 @@ /** * Test-only USTAR writer: the exact archive shape the inspector accepts (pnpm's layout), so a test * can build a package tarball from an in-memory inventory. `package/dist/index.js` is 0755, every - * other entry 0644. + * other entry 0644. The dialect itself is `ustar.mjs`'s. */ -import { Buffer } from 'node:buffer'; +import { ustarEntry, writeChecksum, writeUstar } from './ustar.mjs'; -/** @param {Buffer} header */ -export function checksum(header) { - header.fill(32, 148, 156); - header.write( - header - .reduce((sum, byte) => sum + byte, 0) - .toString(8) - .padStart(6, '0'), - 148, - ); - header[154] = 0; - header[155] = 32; -} +const packageMode = (path) => (path === 'package/dist/index.js' ? 0o755 : 0o644); + +export { writeChecksum as checksum }; /** @param {string} path @param {Uint8Array} bytes */ export function entry(path, bytes) { - const header = Buffer.alloc(512); - header.write(path, 0, 100); - for (const offset of [100, 108, 116, 329, 337]) header.write('000000 \0', offset); - header.write(path === 'package/dist/index.js' ? '000755' : '000644', 100); - header.write(bytes.length.toString(8).padStart(11, '0') + ' ', 124); - header.write('00000000000 ', 136); - header[156] = 48; - header.write('ustar\0' + '00', 257); - checksum(header); - return Buffer.concat([header, bytes, Buffer.alloc((512 - (bytes.length % 512)) % 512)]); + return ustarEntry(path, bytes, packageMode(path)); } /** @param {Map} files */ export function ustar(files) { - return Buffer.concat([ - ...[...files].map(([path, bytes]) => entry(path, bytes)), - Buffer.alloc(1024), - ]); + return writeUstar(files, packageMode); } diff --git a/tools/cli-package/ustar.mjs b/tools/cli-package/ustar.mjs new file mode 100644 index 00000000..56bfdac9 --- /dev/null +++ b/tools/cli-package/ustar.mjs @@ -0,0 +1,164 @@ +/** + * The one strict USTAR dialect the packaging tools read and write (Step 9b.1, the 2026-10-19 C1 ruling): + * regular files only, no linkname, owner strings, prefix, extension records or reserved bytes, zero + * owner ids and device numbers, an exact two-block end and zero padding. The caller decides which + * paths and modes it admits and how a refusal is reported. There is no extraction here and no fallback + * to a more permissive platform tar. + */ +import { Buffer } from 'node:buffer'; +import { TextDecoder } from 'node:util'; + +const BLOCK = 512; +const MAGIC = Buffer.from('ustar\0' + '00'); + +/** @param {Buffer} header */ +function checksumOf(header) { + let sum = 0; + for (let index = 0; index < BLOCK; index += 1) { + sum += index >= 148 && index < 156 ? 32 : (header[index] ?? 0); + } + return sum; +} + +/** @param {Buffer} header */ +export function writeChecksum(header) { + header.fill(32, 148, 156); + header.write(checksumOf(header).toString(8).padStart(6, '0'), 148); + header[154] = 0; + header[155] = 32; +} + +/** + * One regular-file entry: header, bytes and zero padding. + * @param {string} path @param {Uint8Array} bytes @param {number} mode + */ +export function ustarEntry(path, bytes, mode) { + const header = Buffer.alloc(BLOCK); + header.write(path, 0, 100); + for (const offset of [100, 108, 116, 329, 337]) header.write('000000 \0', offset); + header.write(mode.toString(8).padStart(6, '0'), 100); + header.write(bytes.length.toString(8).padStart(11, '0') + ' ', 124); + header.write('00000000000 ', 136); + header[156] = 48; + MAGIC.copy(header, 257); + writeChecksum(header); + return Buffer.concat([header, bytes, Buffer.alloc((BLOCK - (bytes.length % BLOCK)) % BLOCK)]); +} + +/** + * A whole archive in the given entry order. + * @param {Iterable<[string, Uint8Array]>} entries + * @param {(path: string) => number} modeFor + */ +export function writeUstar(entries, modeFor) { + return Buffer.concat([ + ...[...entries].map(([path, bytes]) => ustarEntry(path, bytes, modeFor(path))), + Buffer.alloc(2 * BLOCK), + ]); +} + +/** + * @typedef {{ + * admitPath: (path: string) => void, + * modeFor: (path: string) => number, + * maxFiles: number, + * maxFileBytes: number, + * refuse: (code: string, cause?: unknown) => never, + * admitEntry?: (path: string) => void, + * onFile: (file: {path: string, mode: number, size: number, contents: Buffer}) => void, + * }} UstarReader + */ + +/** + * Read every entry of an uncompressed archive in order. `admitEntry` runs after the header, duplicate + * and case-alias checks and before the entry's bytes are bounded; `onFile` runs once its bytes and + * padding are checked. Returns the entry count. + * @param {Buffer} tar + * @param {UstarReader} reader + */ +export function readUstar(tar, reader) { + const { refuse } = reader; + /** @param {Buffer} field */ + const octal = (field) => { + const text = field.toString('latin1'); + if (!/^[0-7]+[ \0]*$/u.test(text)) refuse('TAR_NUMBER'); + const value = Number.parseInt(text, 8); + if (!Number.isSafeInteger(value)) refuse('TAR_NUMBER'); + return value; + }; + /** @param {Buffer} field */ + const nameField = (field) => { + const end = field.indexOf(0); + if (end < 1 || field.subarray(end).some((byte) => byte !== 0)) refuse('TAR_NAME'); + try { + return new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode( + field.subarray(0, end), + ); + } catch (cause) { + return refuse('UTF8', cause); + } + }; + /** @param {Buffer} header */ + const readHeader = (header) => { + if (octal(header.subarray(148, 156)) !== checksumOf(header)) refuse('TAR_CHECKSUM'); + if (!header.subarray(257, 265).equals(MAGIC)) refuse('TAR_FORMAT'); + if (header[156] !== 48) refuse('TAR_TYPE'); + for (const [start, end] of [ + [157, 257], + [265, 329], + [345, 512], + ]) { + if (header.subarray(start, end).some((byte) => byte !== 0)) refuse('TAR_METADATA'); + } + for (const [start, end] of [ + [108, 116], + [116, 124], + [329, 337], + [337, 345], + ]) { + if (octal(header.subarray(start, end)) !== 0) refuse('TAR_METADATA'); + } + octal(header.subarray(136, 148)); // Validate mtime syntax without requiring a clock value. + const path = nameField(header.subarray(0, 100)); + reader.admitPath(path); + const mode = octal(header.subarray(100, 108)); + if (mode !== reader.modeFor(path)) refuse('TAR_MODE'); + const size = octal(header.subarray(124, 136)); + if (size > reader.maxFileBytes) refuse('FILE_SIZE'); + return { path, mode, size }; + }; + + const seen = new Set(); + const portableNames = new Set(); + let count = 0; + let offset = 0; + let ended = false; + while (offset < tar.length) { + if (tar.length - offset < BLOCK) refuse('TAR_TRUNCATED'); + const header = tar.subarray(offset, offset + BLOCK); + if (header.every((byte) => byte === 0)) { + if (tar.length - offset !== 2 * BLOCK || tar.subarray(offset).some((byte) => byte !== 0)) { + refuse('TAR_END'); + } + ended = true; + break; + } + if (count >= reader.maxFiles) refuse('FILE_COUNT'); + const { path, mode, size } = readHeader(header); + if (seen.has(path)) refuse('DUPLICATE_FILE'); + seen.add(path); + if (portableNames.has(path.toLowerCase())) refuse('PATH_ALIAS'); + portableNames.add(path.toLowerCase()); + reader.admitEntry?.(path); + const start = offset + BLOCK; + const end = start + size; + const next = start + Math.ceil(size / BLOCK) * BLOCK; + if (next > tar.length) refuse('TAR_TRUNCATED'); + if (tar.subarray(end, next).some((byte) => byte !== 0)) refuse('TAR_PADDING'); + reader.onFile({ path, mode, size, contents: tar.subarray(start, end) }); + count += 1; + offset = next; + } + if (!ended) refuse('TAR_END'); + return count; +} diff --git a/tools/cli-package/ustar.test.mjs b/tools/cli-package/ustar.test.mjs new file mode 100644 index 00000000..2f9c1c37 --- /dev/null +++ b/tools/cli-package/ustar.test.mjs @@ -0,0 +1,150 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { Buffer } from 'node:buffer'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { after, test } from 'node:test'; +import { readUstar, ustarEntry, writeChecksum, writeUstar } from './ustar.mjs'; + +const scratch = mkdtempSync(join(tmpdir(), 'relavium-ustar-')); +after(() => rmSync(scratch, { recursive: true, force: true })); + +class Refused extends Error { + constructor(code) { + super(code); + this.code = code; + } +} +const refuses = (code) => (error) => error instanceof Refused && error.code === code; + +/** A private, owner-only dialect like the custody container's. */ +function read(tar, overrides = {}) { + const files = []; + const order = []; + readUstar(tar, { + admitPath: (path) => { + if (!/^(?:[A-Za-z-]+\.json|maps\/[a-z]+\.js\.map)$/u.test(path)) throw new Refused('PATH'); + }, + modeFor: () => 0o600, + maxFiles: 4, + maxFileBytes: 4096, + refuse: (code) => { + throw new Refused(code); + }, + admitEntry: (path) => order.push(`admit ${path}`), + onFile: ({ path, mode, contents }) => { + order.push(`file ${path}`); + files.push([path, mode, Buffer.from(contents)]); + }, + ...overrides, + }); + return { files, order }; +} + +const ENTRIES = [ + ['manifest.json', Buffer.from('{"synthetic":true}\n')], + ['maps/index.js.map', Buffer.from('{"version":3}')], + ['metafile-esm.json', Buffer.alloc(513, 0x61)], +]; + +test('a written archive reads back entry by entry, in order, with its mode', () => { + const { files, order } = read(writeUstar(ENTRIES, () => 0o600)); + assert.deepEqual( + files, + ENTRIES.map(([path, bytes]) => [path, 0o600, bytes]), + ); + assert.deepEqual( + order, + ENTRIES.flatMap(([path]) => [`admit ${path}`, `file ${path}`]), + ); +}); + +test('the platform tar extracts the same bytes, nested directories included', () => { + const archive = join(scratch, 'container.tar'); + const out = join(scratch, 'extracted'); + mkdirSync(out); + writeFileSync( + archive, + writeUstar(ENTRIES, () => 0o600), + ); + const run = spawnSync('tar', ['-xf', archive, '-C', out], { encoding: 'utf8' }); + assert.equal(run.status, 0, 'the platform tar must extract the archive'); + for (const [path, bytes] of ENTRIES) { + assert.deepEqual(readFileSync(join(out, ...path.split('/'))), bytes); + } +}); + +/** One entry archive whose header the mutation edits, with a repaired checksum unless told not to. */ +function mutated(mutate, { repair = true } = {}) { + const block = ustarEntry('manifest.json', Buffer.from('{}'), 0o600); + mutate(block); + if (repair) writeChecksum(block.subarray(0, 512)); + return Buffer.concat([block, Buffer.alloc(1024)]); +} + +test('each header field outside the dialect refuses with its own code', () => { + for (const [code, archive] of [ + ['TAR_CHECKSUM', mutated((block) => (block[0] = 0x4d), { repair: false })], + ['TAR_FORMAT', mutated((block) => block.write('gnutar', 257))], + ['TAR_TYPE', mutated((block) => (block[156] = 0x35))], + ['TAR_METADATA', mutated((block) => (block[157] = 0x61))], + ['TAR_METADATA', mutated((block) => block.write('000001 \0', 108))], + ['TAR_NUMBER', mutated((block) => block.write('00000000009 ', 124))], + ['TAR_NAME', mutated((block) => (block[20] = 0x61))], + ['UTF8', mutated((block) => block.fill(0xff, 0, 4))], + ['TAR_MODE', mutated((block) => block.write('000644', 100))], + ['FILE_SIZE', mutated((block) => block.write('00000010001 ', 124))], + ]) { + assert.throws(() => read(archive), refuses(code), code); + } +}); + +test('the archive structure refuses duplicates, aliases, truncation, padding, count and end', () => { + const one = ustarEntry('manifest.json', Buffer.from('{}'), 0o600); + const end = Buffer.alloc(1024); + const padded = Buffer.from(one); + padded[512 + 2] = 1; + for (const [code, archive] of [ + ['DUPLICATE_FILE', Buffer.concat([one, one, end])], + [ + 'PATH_ALIAS', + Buffer.concat([one, ustarEntry('MANIFEST.json', Buffer.from('{}'), 0o600), end]), + ], + ['TAR_TRUNCATED', one.subarray(0, 600)], + ['TAR_TRUNCATED', Buffer.concat([one.subarray(0, 512), Buffer.alloc(100)])], + ['TAR_PADDING', Buffer.concat([padded, end])], + ['TAR_END', one], + ['TAR_END', Buffer.concat([one, end, Buffer.alloc(512)])], + ['TAR_END', Buffer.concat([one, Buffer.alloc(1023), Buffer.from([1])])], + [ + 'FILE_COUNT', + writeUstar( + ['a', 'b', 'c', 'd', 'e'].map((name) => [`${name}.json`, Buffer.from('{}')]), + () => 0o600, + ), + ], + ]) { + assert.throws(() => read(archive), refuses(code), code); + } + // The caller's path rule runs inside the header check, before the entry's bytes are bounded. + assert.throws( + () => read(Buffer.concat([ustarEntry('outside.txt', Buffer.alloc(0), 0o600), end])), + refuses('PATH'), + ); +}); + +test("the caller's entry hook runs before the entry's bytes are bounded", () => { + const order = []; + assert.throws( + () => + read(writeUstar(ENTRIES, () => 0o600).subarray(0, 600), { + admitEntry: (path) => { + order.push(path); + throw new Refused('EXTRA'); + }, + }), + refuses('EXTRA'), + ); + assert.deepEqual(order, ['manifest.json']); +}); From 27e8e9968d0b5e516bb79f2f10f93df48f748744 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 08:41:34 +0300 Subject: [PATCH 65/97] feat(repo): build the private custody container from the sealed staging Step 9b.1's container is one strict owner-only USTAR of the sealed staging in the staging-relative layout the maintainer approved: finalized.json, manifest.json, metafile-esm.json and maps/.js.map, so a decrypted and extracted container is a staging the manifest reader consumes unchanged. buildPrivateContainer admits only a staging holding exactly those records and the manifest's maps as regular files within the entry bound, every map matching the manifest and the finalizer's record; readPrivateContainer reads the dialect back under the same checks and bounds. Refusals are fixed PACKED_CLI_CUSTODY_CONTAINER_* codes from a new CustodyError. Refs: ADR-0111, Step 9b.1 Co-Authored-By: Claude Opus 5.5 --- package.json | 2 +- tools/cli-package/container.mjs | 201 +++++++++++++++++++++++++++ tools/cli-package/container.test.mjs | 177 +++++++++++++++++++++++ tools/cli-package/custody-error.mjs | 14 ++ 4 files changed, 393 insertions(+), 1 deletion(-) create mode 100644 tools/cli-package/container.mjs create mode 100644 tools/cli-package/container.test.mjs create mode 100644 tools/cli-package/custody-error.mjs diff --git a/package.json b/package.json index b1b4c8e1..0d742813 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "smoke:budget-replay": "node tools/budget-replay-compat/check.mjs", "check:wire-mutations": "node tools/wire-conformance/check.mjs", "check:usage-mutations": "node tools/usage-coverage-controls/check.mjs", - "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/ustar.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/check.test.mjs tools/cli-package/paths.test.mjs tools/cli-package/source-map.test.mjs tools/cli-package/manifest.test.mjs tools/cli-package/symbolicate.test.mjs tools/cli-package/map-controls.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", + "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/ustar.test.mjs tools/cli-package/container.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/check.test.mjs tools/cli-package/paths.test.mjs tools/cli-package/source-map.test.mjs tools/cli-package/manifest.test.mjs tools/cli-package/symbolicate.test.mjs tools/cli-package/map-controls.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", "check:fs-mutations": "node tools/fs-policy-controls/check.mjs", "check:secrets-mutations": "node tools/secrets-controls/check.mjs", "smoke:cli-bootstrap": "node tools/cli-bootstrap/check.mjs", diff --git a/tools/cli-package/container.mjs b/tools/cli-package/container.mjs new file mode 100644 index 00000000..434f483f --- /dev/null +++ b/tools/cli-package/container.mjs @@ -0,0 +1,201 @@ +/** + * The private custody container (Step 9b.1; packaging.md, "Recipient custody" and the private map + * manifest). One strict owner-only USTAR of the sealed staging — `finalized.json`, `manifest.json`, + * `metafile-esm.json` and `maps/.js.map`, staging-relative — so a decrypted, extracted container + * is a staging directory the manifest reader and symbolication consume unchanged. It is built only from + * a staging whose every map is the one the manifest and the finalizer's record name, and read back with + * the same checks. + */ +import { Buffer } from 'node:buffer'; +import { lstatSync, readdirSync } from 'node:fs'; +import { join, posix } from 'node:path'; +import { isDeepStrictEqual } from 'node:util'; +import { refuseCustody } from './custody-error.mjs'; +import { readOrdinaryFile, sha256 } from './files.mjs'; +import { MANIFEST_FILE, validateManifest } from './manifest.mjs'; +import { readUstar, writeUstar } from './ustar.mjs'; + +export const CONTAINER_LIMITS = Object.freeze({ + entryBytes: 16 * 1024 * 1024, + // The three records plus one map for each of the manifest's at most 256 outputs. + entries: 259, + totalBytes: 128 * 1024 * 1024, +}); +const RECORDS = Object.freeze(['finalized.json', MANIFEST_FILE, 'metafile-esm.json']); +const MAP = /^maps\/[A-Za-z0-9_-]+\.js\.map$/u; +const ENTRY_MODE = 0o600; + +/** @param {unknown} value @returns {value is Record} */ +function isRecord(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +/** @param {Buffer} bytes @param {string} code */ +function json(bytes, code) { + try { + return JSON.parse(bytes.toString('utf8')); + } catch { + return refuseCustody(code); + } +} + +/** + * The finalizer's record of `{path, bytes, sha256}` entries, sorted by path. + * @param {unknown} list + * @returns {{path: string, bytes: number, sha256: string}[]} + */ +function recorded(list) { + if ( + !Array.isArray(list) || + !list.every( + (item) => + isRecord(item) && + isDeepStrictEqual(Object.keys(item).sort(), ['bytes', 'path', 'sha256']) && + typeof item.path === 'string' && + Number.isSafeInteger(item.bytes) && + typeof item.sha256 === 'string', + ) + ) { + refuseCustody('CONTAINER_RECORD'); + } + return [...list].sort((left, right) => (left.path < right.path ? -1 : 1)); +} + +/** + * Check a container's entries as a whole: exactly the three records and one map per manifest output, + * each map the manifest's and the finalizer record's, the record naming exactly the manifest's files. + * @param {Map} files + */ +function checkContents(files) { + for (const name of RECORDS) if (!files.has(name)) refuseCustody('CONTAINER_SET'); + const manifest = (() => { + try { + return validateManifest( + json(/** @type {Buffer} */ (files.get(MANIFEST_FILE)), 'CONTAINER_RECORD'), + ); + } catch { + return refuseCustody('CONTAINER_RECORD'); + } + })(); + const maps = manifest.outputs.map((output) => `maps/${posix.basename(output.map.path)}`); + if ( + !isDeepStrictEqual( + [...files.keys()].filter((path) => !RECORDS.includes(path)).sort(), + [...maps].sort(), + ) + ) { + refuseCustody('CONTAINER_SET'); + } + for (const [index, output] of manifest.outputs.entries()) { + if (sha256(/** @type {Buffer} */ (files.get(maps[index]))) !== output.map.sha256) { + refuseCustody('CONTAINER_DRIFT'); + } + } + const record = json(/** @type {Buffer} */ (files.get('finalized.json')), 'CONTAINER_RECORD'); + if ( + !isRecord(record) || + !isDeepStrictEqual(Object.keys(record).sort(), [ + 'maps', + 'outputs', + 'upstreamMaps', + 'version', + ]) || + record.version !== 1 || + !Number.isSafeInteger(record.upstreamMaps) + ) { + refuseCustody('CONTAINER_RECORD'); + } + const outputs = recorded(record.outputs); + const recordedMaps = recorded(record.maps); + if ( + !isDeepStrictEqual( + outputs.map((item) => [item.path, item.sha256]), + manifest.outputs.map((output) => [output.path, output.sha256]), + ) || + !isDeepStrictEqual( + recordedMaps.map((item) => [item.path, item.sha256, item.bytes]), + manifest.outputs.map((output, index) => [ + output.map.path, + output.map.sha256, + /** @type {Buffer} */ (files.get(maps[index])).length, + ]), + ) + ) { + refuseCustody('CONTAINER_RECORD'); + } + return manifest; +} + +/** @param {Map} files */ +function containerBytes(files) { + // A header block per entry, padding to the block, and the two-block end. + return [...files.values()].reduce( + (total, bytes) => total + 512 + Math.ceil(bytes.length / 512) * 512, + 1024, + ); +} + +/** + * Read the sealed staging into one container. The staging must hold exactly the three records and + * `maps/` with exactly the manifest's maps, all regular files within the entry bound. + * @param {string} staging + * @param {typeof CONTAINER_LIMITS} [limits] + * @returns {{bytes: Buffer, manifest: import('./manifest.mjs').PrivateManifest}} + */ +export function buildPrivateContainer(staging, limits = CONTAINER_LIMITS) { + /** @param {string} path */ + const read = (path) => { + try { + return readOrdinaryFile(path, limits.entryBytes); + } catch { + return refuseCustody('CONTAINER_ENTRY'); + } + }; + if (!isDeepStrictEqual(readdirSync(staging).sort(), [...RECORDS, 'maps'].sort())) { + refuseCustody('CONTAINER_SET'); + } + const mapsDirectory = lstatSync(join(staging, 'maps')); + if (!mapsDirectory.isDirectory() || mapsDirectory.isSymbolicLink()) { + refuseCustody('CONTAINER_SET'); + } + const names = readdirSync(join(staging, 'maps')).sort(); + if (names.length > limits.entries - RECORDS.length) refuseCustody('CONTAINER_SIZE'); + /** @type {Map} */ + const files = new Map(); + for (const name of RECORDS) files.set(name, read(join(staging, name))); + for (const name of names) { + if (!MAP.test(`maps/${name}`)) refuseCustody('CONTAINER_SET'); + files.set(`maps/${name}`, read(join(staging, 'maps', name))); + } + const manifest = checkContents(files); + if (containerBytes(files) > limits.totalBytes) refuseCustody('CONTAINER_SIZE'); + const sorted = [...files].sort(([left], [right]) => (left < right ? -1 : 1)); + const bytes = writeUstar(sorted, () => ENTRY_MODE); + // The written container must read back as exactly these entries. + const again = readPrivateContainer(bytes, limits); + if (!isDeepStrictEqual([...again.files], sorted)) refuseCustody('CONTAINER_SELF_CHECK'); + return { bytes, manifest }; +} + +/** + * Read and check a container: the strict owner-only dialect, the bounds, and the contents as a whole. + * @param {Buffer} bytes + * @param {typeof CONTAINER_LIMITS} [limits] + * @returns {{files: Map, manifest: import('./manifest.mjs').PrivateManifest}} + */ +export function readPrivateContainer(bytes, limits = CONTAINER_LIMITS) { + if (!Buffer.isBuffer(bytes) || bytes.length > limits.totalBytes) refuseCustody('CONTAINER_SIZE'); + /** @type {Map} */ + const files = new Map(); + readUstar(bytes, { + admitPath: (path) => { + if (!RECORDS.includes(path) && !MAP.test(path)) refuseCustody('CONTAINER_PATH'); + }, + modeFor: () => ENTRY_MODE, + maxFiles: limits.entries, + maxFileBytes: limits.entryBytes, + refuse: (code) => refuseCustody(`CONTAINER_${code}`), + onFile: ({ path, contents }) => files.set(path, Buffer.from(contents)), + }); + return { files, manifest: checkContents(files) }; +} diff --git a/tools/cli-package/container.test.mjs b/tools/cli-package/container.test.mjs new file mode 100644 index 00000000..a0645ffd --- /dev/null +++ b/tools/cli-package/container.test.mjs @@ -0,0 +1,177 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { Buffer } from 'node:buffer'; +import { + cpSync, + mkdirSync, + readFileSync, + readdirSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { join } from 'node:path'; +import { before, test } from 'node:test'; +import { candidate, fresh, posixModes, seal } from './candidate-fixture.mjs'; +import { CONTAINER_LIMITS, buildPrivateContainer, readPrivateContainer } from './container.mjs'; +import { CustodyError } from './custody-error.mjs'; +import { readPrivateManifest } from './manifest.mjs'; +import { writeUstar } from './ustar.mjs'; + +const refuses = (code) => (error) => error instanceof CustodyError && error.code === code; + +/** @type {string} */ +let sealed; +before(async () => { + // One synthetic candidate sealed as the producer seals it, plus the metafile the finalizer stages. + const fixture = await candidate(); + await seal(fixture); + writeFileSync(join(fixture.staging, 'metafile-esm.json'), '{"inputs":{},"outputs":{}}'); + sealed = fixture.staging; +}); + +/** A fresh copy of the sealed staging, changed by `change`. */ +function staging(change = () => {}) { + const copy = fresh('container-staging'); + cpSync(sealed, copy, { recursive: true }); + change(copy); + return copy; +} + +const mapName = (copy) => readdirSync(join(copy, 'maps'))[0]; + +test('the sealed staging becomes one container that reads back and extracts to the same staging', () => { + const copy = staging(); + const { bytes, manifest } = buildPrivateContainer(copy); + assert.deepEqual(manifest, readPrivateManifest(copy)); + const read = readPrivateContainer(bytes); + assert.deepEqual(read.manifest, manifest); + const expected = [ + 'finalized.json', + 'manifest.json', + ...readdirSync(join(copy, 'maps')) + .sort() + .map((name) => `maps/${name}`), + 'metafile-esm.json', + ]; + assert.deepEqual([...read.files.keys()], expected); + // Recovery is `gpg -d | tar -x`: the platform tar's extraction is a staging the manifest reader reads. + const archive = join(fresh('container-archive'), 'container.tar'); + mkdirSync(join(archive, '..')); + writeFileSync(archive, bytes); + const out = fresh('container-extracted'); + mkdirSync(out); + assert.equal(spawnSync('tar', ['-xf', archive, '-C', out]).status, 0); + for (const path of expected) { + assert.deepEqual( + readFileSync(join(out, ...path.split('/'))), + readFileSync(join(copy, ...path.split('/'))), + ); + } + assert.deepEqual(readPrivateManifest(out), manifest); +}); + +test('a staging that is not exactly the sealed records and maps refuses', () => { + for (const [code, change] of [ + ['CONTAINER_SET', (copy) => writeFileSync(join(copy, 'notes.txt'), 'extra')], + ['CONTAINER_SET', (copy) => rmSync(join(copy, 'metafile-esm.json'))], + ['CONTAINER_SET', (copy) => rmSync(join(copy, 'manifest.json'))], + ['CONTAINER_SET', (copy) => writeFileSync(join(copy, 'maps', 'extra.js.map'), '{}')], + ['CONTAINER_SET', (copy) => rmSync(join(copy, 'maps', mapName(copy)))], + ['CONTAINER_SET', (copy) => writeFileSync(join(copy, 'maps', 'notes.txt'), '{}')], + [ + 'CONTAINER_DRIFT', + (copy) => { + const path = join(copy, 'maps', mapName(copy)); + writeFileSync(path, Buffer.concat([readFileSync(path), Buffer.from(' ')])); + }, + ], + [ + 'CONTAINER_RECORD', + (copy) => { + const path = join(copy, 'finalized.json'); + const record = JSON.parse(readFileSync(path, 'utf8')); + record.outputs[0].sha256 = '0'.repeat(64); + writeFileSync(path, JSON.stringify(record)); + }, + ], + ['CONTAINER_RECORD', (copy) => writeFileSync(join(copy, 'manifest.json'), '{"version":1}')], + ]) { + assert.throws(() => buildPrivateContainer(staging(change)), refuses(code), code); + } +}); + +test('a linked record or maps directory refuses', { skip: !posixModes }, () => { + assert.throws( + () => + buildPrivateContainer( + staging((copy) => { + const target = `${fresh('linked-metafile')}.json`; + writeFileSync(target, '{}'); + rmSync(join(copy, 'metafile-esm.json')); + symlinkSync(target, join(copy, 'metafile-esm.json')); + }), + ), + refuses('CONTAINER_ENTRY'), + ); + assert.throws( + () => + buildPrivateContainer( + staging((copy) => { + const maps = fresh('linked-maps'); + cpSync(join(copy, 'maps'), maps, { recursive: true }); + rmSync(join(copy, 'maps'), { recursive: true }); + symlinkSync(maps, join(copy, 'maps')); + }), + ), + refuses('CONTAINER_SET'), + ); +}); + +test('the entry, count and total bounds refuse before a container is written', () => { + const copy = staging(); + const largest = Math.max( + ...['finalized.json', 'manifest.json', 'metafile-esm.json'].map( + (name) => readFileSync(join(copy, name)).length, + ), + ...readdirSync(join(copy, 'maps')).map((name) => readFileSync(join(copy, 'maps', name)).length), + ); + assert.throws( + () => buildPrivateContainer(copy, { ...CONTAINER_LIMITS, entryBytes: largest - 1 }), + refuses('CONTAINER_ENTRY'), + ); + assert.throws( + () => buildPrivateContainer(copy, { ...CONTAINER_LIMITS, entries: 3 }), + refuses('CONTAINER_SIZE'), + ); + const { bytes } = buildPrivateContainer(copy); + assert.throws( + () => buildPrivateContainer(copy, { ...CONTAINER_LIMITS, totalBytes: bytes.length - 1 }), + refuses('CONTAINER_SIZE'), + ); + assert.throws( + () => readPrivateContainer(bytes, { ...CONTAINER_LIMITS, totalBytes: bytes.length - 1 }), + refuses('CONTAINER_SIZE'), + ); +}); + +test('a container outside the owner-only dialect, or whose contents disagree, refuses on reading', () => { + const files = readPrivateContainer(buildPrivateContainer(staging()).bytes).files; + const write = (entries, mode = 0o600) => writeUstar(entries, () => mode); + assert.throws( + () => readPrivateContainer(write([...files, ['notes.txt', Buffer.from('x')]])), + refuses('CONTAINER_PATH'), + ); + assert.throws( + () => readPrivateContainer(write([...files], 0o644)), + refuses('CONTAINER_TAR_MODE'), + ); + const [map] = [...files.keys()].filter((path) => path.startsWith('maps/')); + const drifted = new Map(files); + drifted.set(map, Buffer.concat([files.get(map), Buffer.from(' ')])); + assert.throws(() => readPrivateContainer(write([...drifted])), refuses('CONTAINER_DRIFT')); + const missing = new Map(files); + missing.delete('metafile-esm.json'); + assert.throws(() => readPrivateContainer(write([...missing])), refuses('CONTAINER_SET')); + assert.throws(() => readPrivateContainer(Uint8Array.from([1])), refuses('CONTAINER_SIZE')); +}); diff --git a/tools/cli-package/custody-error.mjs b/tools/cli-package/custody-error.mjs new file mode 100644 index 00000000..408cb728 --- /dev/null +++ b/tools/cli-package/custody-error.mjs @@ -0,0 +1,14 @@ +/** The custody tools' one refusal (Step 9b.1): a fixed code, never a path, key, map byte or child output. */ +export class CustodyError extends Error { + /** @param {string} code */ + constructor(code) { + super(`PACKED_CLI_CUSTODY_${code}`); + this.name = 'CustodyError'; + this.code = code; + } +} + +/** @param {string} code @returns {never} */ +export function refuseCustody(code) { + throw new CustodyError(code); +} From f3f5b320255e4055a7e28debc6857f7c7bb3b872 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 08:56:05 +0300 Subject: [PATCH 66/97] feat(repo): add the isolated GnuPG runner for recipient custody GnuPG is release tooling, never a runtime dependency (packaging.md, "Recipient custody"). gnupg.mjs resolves gpg from absolute PATH entries only, refuses a missing binary or one below GnuPG 2.2.8, creates each owner-only home itself and runs every call isolated: --no-options, --no-autostart and --no-keyring, in the C locale with an isolated environment, bounded through runCommand. The argv builders list a recipient file with the default trust model (GnuPG 2.4 cannot list an expired or revoked key under --trust-model always without a trust database) and encrypt with --trust-model always, AES-256 preference, no compression and a fixed literal name; only status lines and colon records are parsed, and a home may hold only the regular files GnuPG creates for these calls. GnuPG >= 2.2.8 joins the local prerequisites: the custody tests fail rather than skip without it. Refs: ADR-0111, Step 9b.1 Co-Authored-By: Claude Opus 5.5 --- docs/runbooks/local-dev-setup.md | 1 + package.json | 2 +- tools/cli-package/gnupg.mjs | 192 +++++++++++++++++++++++++++++++ tools/cli-package/gnupg.test.mjs | 161 ++++++++++++++++++++++++++ 4 files changed, 355 insertions(+), 1 deletion(-) create mode 100644 tools/cli-package/gnupg.mjs create mode 100644 tools/cli-package/gnupg.test.mjs diff --git a/docs/runbooks/local-dev-setup.md b/docs/runbooks/local-dev-setup.md index 14998fe3..de824a50 100644 --- a/docs/runbooks/local-dev-setup.md +++ b/docs/runbooks/local-dev-setup.md @@ -21,6 +21,7 @@ For the full package layout this runbook references, see | Rust toolchain (`rustup`, `cargo`) | Builds the Tauri v2 desktop backend | Only needed to build `apps/desktop`. The engine, CLI, and tests do not need Rust. | | Tauri OS prerequisites | Native WebView + build deps | macOS: Xcode Command Line Tools. Windows: WebView2 + MSVC build tools. Linux: WebKitGTK + libsecret dev packages. See the [Tauri prerequisites guide](https://v2.tauri.app/start/prerequisites/). | | Git | Workflows and agents are git-native YAML | — | +| GnuPG ≥ 2.2.8 (`gpg`) | The release custody tests (`pnpm run ci`) encrypt synthetic map containers | Required, never skipped: the tests fail without it. macOS: `brew install gnupg`. Debian/Ubuntu: `apt install gnupg` (the CI Ubuntu runner has it). Windows: Gpg4win; the CI runs these tests on Ubuntu only. Release tooling, never a runtime dependency. | > You can do meaningful work on the engine and CLI **without** the Rust/Tauri > toolchain. Install Rust only when you start on `apps/desktop`. diff --git a/package.json b/package.json index 0d742813..bfdd15db 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "smoke:budget-replay": "node tools/budget-replay-compat/check.mjs", "check:wire-mutations": "node tools/wire-conformance/check.mjs", "check:usage-mutations": "node tools/usage-coverage-controls/check.mjs", - "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/ustar.test.mjs tools/cli-package/container.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/check.test.mjs tools/cli-package/paths.test.mjs tools/cli-package/source-map.test.mjs tools/cli-package/manifest.test.mjs tools/cli-package/symbolicate.test.mjs tools/cli-package/map-controls.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", + "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/ustar.test.mjs tools/cli-package/container.test.mjs tools/cli-package/gnupg.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/check.test.mjs tools/cli-package/paths.test.mjs tools/cli-package/source-map.test.mjs tools/cli-package/manifest.test.mjs tools/cli-package/symbolicate.test.mjs tools/cli-package/map-controls.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", "check:fs-mutations": "node tools/fs-policy-controls/check.mjs", "check:secrets-mutations": "node tools/secrets-controls/check.mjs", "smoke:cli-bootstrap": "node tools/cli-bootstrap/check.mjs", diff --git a/tools/cli-package/gnupg.mjs b/tools/cli-package/gnupg.mjs new file mode 100644 index 00000000..11031ad2 --- /dev/null +++ b/tools/cli-package/gnupg.mjs @@ -0,0 +1,192 @@ +/** + * The isolated GnuPG boundary of the custody tools (Step 9b.1; packaging.md, "Recipient custody"). + * GnuPG is release tooling, never a runtime dependency. Every call runs the resolved absolute binary + * through the shell-free, bounded `runCommand`, in a fresh owner-only home the tools create themselves, + * with no options file, keyring, agent or key discovery. Only status lines and colon records are + * parsed; human-readable output and stderr are never read or surfaced. + */ +import { existsSync, lstatSync, mkdirSync, readdirSync, realpathSync, statSync } from 'node:fs'; +import { delimiter, isAbsolute, join } from 'node:path'; +import { refuseCustody } from './custody-error.mjs'; +import { isolatedEnvironment } from './process.mjs'; + +/** GnuPG 2.2.8 has `--show-keys`, the CVE-2018-12020 fix and refuses missing-MDC decryption. */ +export const GNUPG_FLOOR = Object.freeze([2, 2, 8]); +/** Bounds for one GnuPG call: its status and colon output are small. */ +export const GNUPG_CALL = Object.freeze({ timeoutMs: 60_000, maxBytes: 256 * 1024 }); +/** The literal-packet name inside the ciphertext, so no host file name enters it. */ +export const LITERAL_NAME = 'relavium-private-maps.tar'; + +/** + * Resolve `gpg` from absolute PATH entries only, without a shell. + * @returns {{command: string, prefix: string[]}} + */ +export function resolveGpg(path = process.env.PATH ?? '', platform = process.platform) { + for (const directory of path.split(delimiter).filter(Boolean)) { + if (!isAbsolute(directory)) continue; + const candidate = join(directory, platform === 'win32' ? 'gpg.exe' : 'gpg'); + if (existsSync(candidate) && statSync(candidate).isFile()) { + return { command: realpathSync(candidate), prefix: [] }; + } + } + return refuseCustody('GPG_UNAVAILABLE'); +} + +/** + * A fresh owner-only GnuPG home. Created here, so no `gpg.conf`, `common.conf` or keyring can already + * be in it. + * @param {string} parent @param {string} name + */ +export function createGnupgHome(parent, name) { + const home = join(parent, name); + mkdirSync(home, { mode: 0o700 }); + return home; +} + +/** The isolated environment for GnuPG, in the C locale, with every home and temporary path in `work`. */ +export function gnupgEnvironment(work) { + return { ...isolatedEnvironment(work), LANG: 'C', LC_ALL: 'C' }; +} + +/** @param {string} home */ +const isolated = (home) => [ + '--homedir', + home, + '--batch', + '--no-tty', + '--no-options', + '--no-autostart', + '--no-keyring', +]; + +/** + * List the keys in one recipient file as colon records, importing nothing. The default trust model + * runs here: GnuPG 2.4 cannot list an expired or revoked key under `--trust-model always` without a + * trust database, so this home may gain an empty one; encryption runs in a separate home. + * @param {string} home @param {string} recipient + */ +export function listArgs(home, recipient) { + return [ + ...isolated(home), + '--with-colons', + '--fixed-list-mode', + '--with-subkey-fingerprint', + '--show-keys', + recipient, + ]; +} + +/** + * Encrypt `input` to the one key in `recipient`, its status on stdout. `--trust-model always`: the + * assurance is the pinned fingerprints and the colon pre-check, and GnuPG treats a `--recipient-file` + * key as fully valid. No compression keeps the ciphertext's size bound. + * @param {string} home @param {string} recipient @param {string} input @param {string} output + */ +export function encryptArgs(home, recipient, input, output) { + return [ + ...isolated(home), + '--no-encrypt-to', + '--no-auto-key-locate', + '--no-auto-key-retrieve', + '--trust-model', + 'always', + '--personal-cipher-preferences', + 'AES256', + '--compress-algo', + 'none', + '--status-fd', + '1', + '--set-filename', + LITERAL_NAME, + '--output', + output, + '--encrypt', + '--recipient-file', + recipient, + input, + ]; +} + +/** + * The recipients of a ciphertext, from its public-key packets: needs no secret key. + * @param {string} home @param {string} ciphertext + */ +export function recipientsArgs(home, ciphertext) { + return [ + ...isolated(home), + '--trust-model', + 'always', + '--status-fd', + '1', + '--list-only', + '--decrypt', + ciphertext, + ]; +} + +/** + * Status lines only: any other line refuses. Unknown keywords and extra arguments are kept for the + * caller to ignore, as GnuPG's DETAILS requires. + * @param {string} text + * @returns {{keyword: string, args: string[]}[]} + */ +export function parseStatus(text) { + return text + .split('\n') + .filter((line, index, lines) => !(index === lines.length - 1 && line === '')) + .map((line) => { + const match = /^\[GNUPG:\] ([A-Z_]+)(?: (.*))?$/u.exec(line); + if (!match) return refuseCustody('STATUS'); + return { keyword: match[1], args: match[2] === undefined ? [] : match[2].split(' ') }; + }); +} + +/** + * Colon records, one array of fields per line. + * @param {string} text + * @returns {string[][]} + */ +export function parseColons(text) { + return text + .split('\n') + .filter((line) => line !== '') + .map((line) => line.split(':')); +} + +/** + * The GnuPG version, refused below the floor. + * @param {{command: string, prefix: string[]}} gpg + * @param {{run: Function, env: Record, home: string}} options + */ +export async function gnupgVersion(gpg, { run, env, home }) { + let stdout; + try { + ({ stdout } = await run(gpg.command, [...gpg.prefix, '--homedir', home, '--version'], { + ...GNUPG_CALL, + env, + })); + } catch { + return refuseCustody('GPG_UNAVAILABLE'); + } + const match = /^gpg \(GnuPG(?:\/[^)]+)?\) (\d+)\.(\d+)\.(\d+)\s/u.exec(String(stdout)); + if (!match) return refuseCustody('GPG_VERSION'); + const version = match.slice(1, 4).map(Number); + for (const [index, floor] of GNUPG_FLOOR.entries()) { + if (version[index] > floor) break; + if (version[index] < floor) refuseCustody('GPG_VERSION'); + } + return version.join('.'); +} + +/** + * The home may hold only regular files GnuPG itself creates for these calls: never a socket, a + * configuration, a keyring or private-key material. + * @param {string} home @param {readonly string[]} allowed + */ +export function assertHomeState(home, allowed) { + for (const name of readdirSync(home)) { + if (!allowed.includes(name) || !lstatSync(join(home, name)).isFile()) { + refuseCustody('HOME_STATE'); + } + } +} diff --git a/tools/cli-package/gnupg.test.mjs b/tools/cli-package/gnupg.test.mjs new file mode 100644 index 00000000..f15b2913 --- /dev/null +++ b/tools/cli-package/gnupg.test.mjs @@ -0,0 +1,161 @@ +import assert from 'node:assert/strict'; +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { delimiter, join } from 'node:path'; +import { after, test } from 'node:test'; +import { CustodyError } from './custody-error.mjs'; +import { + GNUPG_FLOOR, + assertHomeState, + createGnupgHome, + encryptArgs, + gnupgEnvironment, + gnupgVersion, + listArgs, + parseColons, + parseStatus, + recipientsArgs, + resolveGpg, +} from './gnupg.mjs'; +import { runCommand } from './process.mjs'; + +const scratch = mkdtempSync(join(tmpdir(), 'relavium-gnupg-')); +after(() => rmSync(scratch, { recursive: true, force: true })); +const refuses = (code) => (error) => error instanceof CustodyError && error.code === code; + +test('gpg resolves from absolute PATH entries only and is required, never skipped', async () => { + const gpg = resolveGpg(); + assert.ok(gpg.command.length > 0 && gpg.prefix.length === 0); + // The real binary meets the floor; this is the prerequisite local `pnpm run ci` and CI rely on. + const home = createGnupgHome(scratch, 'version'); + const version = await gnupgVersion(gpg, { + run: runCommand, + env: gnupgEnvironment(scratch), + home, + }); + assert.match(version, /^\d+\.\d+\.\d+$/u); + assert.throws(() => resolveGpg(join(scratch, 'none')), refuses('GPG_UNAVAILABLE')); + // A relative entry is never searched, even when it holds a gpg. + mkdirSync(join(scratch, 'relative')); + writeFileSync(join(scratch, 'relative', 'gpg'), ''); + assert.throws( + () => resolveGpg(['relative', join(scratch, 'none')].join(delimiter)), + refuses('GPG_UNAVAILABLE'), + ); +}); + +test('the version floor is GnuPG 2.2.8, read from the version line only', async () => { + assert.deepEqual(GNUPG_FLOOR, [2, 2, 8]); + const gpg = { command: 'gpg', prefix: [] }; + const version = (stdout) => + gnupgVersion(gpg, { run: async () => ({ stdout, stderr: '' }), env: {}, home: scratch }); + for (const line of [ + 'gpg (GnuPG) 2.2.8\n', + 'gpg (GnuPG/MacGPG2) 2.2.41\n', + 'gpg (GnuPG) 2.4.4\n', + ]) { + assert.match(await version(line), /^2\.\d+\.\d+$/u); + } + for (const line of ['gpg (GnuPG) 2.2.7\n', 'gpg (GnuPG) 2.1.23\n', 'gpg (GnuPG) 1.4.23\n']) { + await assert.rejects(version(line), refuses('GPG_VERSION')); + } + await assert.rejects(version('gpg 2.4.4\n'), refuses('GPG_VERSION')); + await assert.rejects( + gnupgVersion(gpg, { + run: async () => { + throw new Error('spawn failed'); + }, + env: {}, + home: scratch, + }), + refuses('GPG_UNAVAILABLE'), + ); +}); + +test('every call is isolated, and encryption never passes a flag that widens it', () => { + const required = [ + '--homedir', + '--batch', + '--no-tty', + '--no-options', + '--no-autostart', + '--no-keyring', + ]; + const forbidden = [ + '--yes', + '-R', + '-F', + '-r', + '--recipient', + '--hidden-recipient', + '--throw-keyids', + '--faked-system-time', + '--ignore-time-conflict', + '--ignore-valid-from', + '--keyring', + '--import', + '--encrypt-to', + ]; + const calls = [listArgs('H', 'K'), encryptArgs('H', 'K', 'I', 'O'), recipientsArgs('H', 'C')]; + for (const args of calls) { + for (const flag of required) assert.ok(args.includes(flag), flag); + assert.equal(args[args.indexOf('--homedir') + 1], 'H'); + for (const flag of forbidden) assert.ok(!args.includes(flag), flag); + assert.ok(!args.some((arg) => arg.startsWith('--allow-'))); + } + const encrypt = encryptArgs('H', 'K', 'I', 'O'); + for (const [flag, value] of [ + ['--trust-model', 'always'], + ['--personal-cipher-preferences', 'AES256'], + ['--compress-algo', 'none'], + ['--status-fd', '1'], + ['--set-filename', 'relavium-private-maps.tar'], + ['--output', 'O'], + ['--recipient-file', 'K'], + ]) { + assert.equal(encrypt[encrypt.indexOf(flag) + 1], value, flag); + } + for (const flag of ['--no-encrypt-to', '--no-auto-key-locate', '--no-auto-key-retrieve']) { + assert.ok(encrypt.includes(flag), flag); + } + assert.equal(encrypt.at(-1), 'I'); +}); + +test('status parsing admits only status lines and keeps unknown keywords for the caller', () => { + assert.deepEqual(parseStatus('[GNUPG:] BEGIN_ENCRYPTION 2 9\n[GNUPG:] NEWSIG\n'), [ + { keyword: 'BEGIN_ENCRYPTION', args: ['2', '9'] }, + { keyword: 'NEWSIG', args: [] }, + ]); + assert.deepEqual(parseStatus(''), []); + for (const text of [ + 'gpg: encrypted\n', + '[GNUPG:] begin\n', + '[GNUPG:] BEGIN_ENCRYPTION 2 9\n\n', + '[GNUPG:]BEGIN_ENCRYPTION\n', + ]) { + assert.throws(() => parseStatus(text), refuses('STATUS'), text); + } + assert.deepEqual(parseColons('pub:-:255\nfpr:::\n'), [ + ['pub', '-', '255'], + ['fpr', '', '', ''], + ]); +}); + +test('a home may hold only the regular files GnuPG creates for these calls', () => { + const home = createGnupgHome(scratch, 'state'); + assertHomeState(home, ['random_seed']); + writeFileSync(join(home, 'random_seed'), ''); + assertHomeState(home, ['random_seed']); + for (const [index, plant] of [ + (path) => writeFileSync(join(path, 'common.conf'), 'use-keyboxd\n'), + (path) => mkdirSync(join(path, 'private-keys-v1.d')), + (path) => symlinkSync(join(scratch, 'elsewhere'), join(path, 'S.gpg-agent')), + ].entries()) { + const planted = createGnupgHome(scratch, `planted-${index}`); + plant(planted); + assert.throws(() => assertHomeState(planted, ['random_seed']), refuses('HOME_STATE')); + } + const linkedSeed = createGnupgHome(scratch, 'linked-seed'); + symlinkSync(join(scratch, 'elsewhere'), join(linkedSeed, 'random_seed')); + assert.throws(() => assertHomeState(linkedSeed, ['random_seed']), refuses('HOME_STATE')); +}); From e0d39a492e87cd5f426a7f432772720320b79645 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 08:56:05 +0300 Subject: [PATCH 67/97] test(repo): commit the synthetic custody fixtures Armored public OpenPGP v4 keys under fixture.invalid for every recipient class Step 9b.1 refuses or admits: good and other (cv25519, no AEAD feature), good-aead, rsa3072 and rsa2048, a primary and a subkey expired in 2020, a revoked primary and a revoked subkey, a sign-only key and a key with two encryption subkeys, with their pins. A maintainer-only generator, never run in CI, wrote them from a throwaway home (the only place that may fake the system time) and removed it with its agent stopped. No secret key is committed: gpg-fixture.mjs generates the round-trip decryption key per run in a short owner-only home and stops its agent afterwards. Refs: ADR-0111, Step 9b.1 Co-Authored-By: Claude Opus 5.5 --- .../custody-fixtures/expired-primary.asc | 15 ++ .../custody-fixtures/expired-subkey.asc | 15 ++ .../cli-package/custody-fixtures/generate.mjs | 155 ++++++++++++++++++ .../custody-fixtures/good-aead.asc | 14 ++ tools/cli-package/custody-fixtures/good.asc | 14 ++ tools/cli-package/custody-fixtures/other.asc | 14 ++ tools/cli-package/custody-fixtures/pins.json | 46 ++++++ .../custody-fixtures/revoked-primary.asc | 18 ++ .../custody-fixtures/revoked-subkey.asc | 17 ++ .../cli-package/custody-fixtures/rsa2048.asc | 39 +++++ .../cli-package/custody-fixtures/rsa3072.asc | 42 +++++ .../custody-fixtures/sign-only.asc | 10 ++ .../two-encryption-subkeys.asc | 19 +++ tools/cli-package/gpg-fixture.mjs | 100 +++++++++++ 14 files changed, 518 insertions(+) create mode 100644 tools/cli-package/custody-fixtures/expired-primary.asc create mode 100644 tools/cli-package/custody-fixtures/expired-subkey.asc create mode 100644 tools/cli-package/custody-fixtures/generate.mjs create mode 100644 tools/cli-package/custody-fixtures/good-aead.asc create mode 100644 tools/cli-package/custody-fixtures/good.asc create mode 100644 tools/cli-package/custody-fixtures/other.asc create mode 100644 tools/cli-package/custody-fixtures/pins.json create mode 100644 tools/cli-package/custody-fixtures/revoked-primary.asc create mode 100644 tools/cli-package/custody-fixtures/revoked-subkey.asc create mode 100644 tools/cli-package/custody-fixtures/rsa2048.asc create mode 100644 tools/cli-package/custody-fixtures/rsa3072.asc create mode 100644 tools/cli-package/custody-fixtures/sign-only.asc create mode 100644 tools/cli-package/custody-fixtures/two-encryption-subkeys.asc create mode 100644 tools/cli-package/gpg-fixture.mjs diff --git a/tools/cli-package/custody-fixtures/expired-primary.asc b/tools/cli-package/custody-fixtures/expired-primary.asc new file mode 100644 index 00000000..33d7f7d1 --- /dev/null +++ b/tools/cli-package/custody-fixtures/expired-primary.asc @@ -0,0 +1,15 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEXgvhABYJKwYBBAHaRw8BAQdA4EjzLwY9BoBLpq5V0JHMtZxzJXpNVuK+FSiV +ujKVQzu0MWV4cGlyZWQtcHJpbWFyeSA8ZXhwaXJlZC1wcmltYXJ5QGZpeHR1cmUu +aW52YWxpZD6ItQQTFgoAXRYhBOU2Am1dlOSxXumEAFcN+Q5G2HLdBQJeC+EAGxSA +AAAAAAQADm1hbnUyLDIuNSsxLjEyLDAsMwIbAQUJAAFRgAULCQgHAgIiAgYVCgkI +CwIEFgIDAQIeBwIXgAAKCRBXDfkORthy3fGTAP97iZY7uSZCmDjGdzDTxICFAxcz +08w4FTgMt4MzNq/7JwEA1mX7T+bkq3U3FGQ2ZH3C5JNen041M/WwuS+vYxhDxQO4 +OAReC+EAEgorBgEEAZdVAQUBAQdAEUr8zfwnwvMx8WF4bB6gp1uXpGaL6VNDfi5e +zws9+WADAQgHiJQEGBYKADwWIQTlNgJtXZTksV7phABXDfkORthy3QUCXgvhABsU +gAAAAAAEAA5tYW51MiwyLjUrMS4xMiwwLDMCGwwACgkQVw35DkbYct0epQEAgcTs +Ew7PUofTu7gorjBuJwJKD/KClhgVfwFwJ3GYEW8BAI1xbQo1YLQC4Zv5fFNgZ/9L +kTHVZmTjEzLP32pdp6gC +=mAOL +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tools/cli-package/custody-fixtures/expired-subkey.asc b/tools/cli-package/custody-fixtures/expired-subkey.asc new file mode 100644 index 00000000..110a6b29 --- /dev/null +++ b/tools/cli-package/custody-fixtures/expired-subkey.asc @@ -0,0 +1,15 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEXgvhABYJKwYBBAHaRw8BAQdAT7eN1lZthk5wYgrMnitVnGNPnEsHrYODB0Cu +To0VmuO0L2V4cGlyZWQtc3Via2V5IDxleHBpcmVkLXN1YmtleUBmaXh0dXJlLmlu +dmFsaWQ+iK8EExYKAFcWIQTumbKL6S4sns/QUEN7BuSkoQP2OQUCXgvhABsUgAAA +AAAEAA5tYW51MiwyLjUrMS4xMiwwLDMCGwEFCwkIBwICIgIGFQoJCAsCBBYCAwEC +HgcCF4AACgkQewbkpKED9jlLUAD9HVjUpkciDwaN4d1Uh1E7G7IrktHgf7E1FoPU +70a6iIMA/3t1NSUSlUE68tvdvlCohePX61k4o057Gw3cXUHpXVcAuDgEXgvhABIK +KwYBBAGXVQEFAQEHQBsWrx9liPO7pusMyDmq+LKF4M9W/lUssiD/IEksTtZEAwEI +B4iaBBgWCgBCFiEE7pmyi+kuLJ7P0FBDewbkpKED9jkFAl4L4QAbFIAAAAAABAAO +bWFudTIsMi41KzEuMTIsMCwzAhsMBQkAAVGAAAoJEHsG5KShA/Y52t8BALLdyWEr +bX0wKNvONJrsddx3JrXEayrpeTVqba6KM1+sAP9EcTa2tdIfK1s58sEFQEkw+l4d +cuz9Osg6pZ4hEJvvCg== +=BA/e +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tools/cli-package/custody-fixtures/generate.mjs b/tools/cli-package/custody-fixtures/generate.mjs new file mode 100644 index 00000000..373bc577 --- /dev/null +++ b/tools/cli-package/custody-fixtures/generate.mjs @@ -0,0 +1,155 @@ +/** + * Maintainer-only generator of the committed synthetic custody fixtures (Step 9b.1). It is never run in + * CI: generation is not deterministic, so the armored public keys and `pins.json` it writes are + * committed and reviewed instead. Every key is a synthetic OpenPGP v4 key under `fixture.invalid`; no + * secret key leaves the throwaway home, which is removed with its agent stopped. Only this generator + * may fake the system time, to create keys that expired in 2020. + * + * node tools/cli-package/custody-fixtures/generate.mjs + */ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const here = fileURLToPath(new URL('.', import.meta.url)); +// A short home: the agent's socket path must fit the platform's limit (104 bytes on macOS). +const home = mkdtempSync(join(tmpdir(), 'rg-')); +const PAST = '20200101T000000!'; +const NO_AEAD = 'AES256 AES192 AES SHA512 SHA384 SHA256 ZLIB ZIP Uncompressed'; + +/** @param {string[]} args @param {{input?: string, past?: boolean}} [options] */ +function gpg(args, { input, past = false } = {}) { + const run = spawnSync( + 'gpg', + [ + '--homedir', + home, + '--batch', + '--no-tty', + '--pinentry-mode', + 'loopback', + '--passphrase', + '', + ...(past ? ['--faked-system-time', PAST] : []), + ...args, + ], + { encoding: 'utf8', input, env: { PATH: process.env.PATH, GNUPGHOME: home, LANG: 'C' } }, + ); + assert.equal(run.status, 0, `gpg ${args.join(' ')} failed:\n${run.stderr}`); + return run.stdout; +} + +/** The primary and subkey fingerprints, in listing order, of the key with this UID. */ +function fingerprints(uid) { + const fprs = []; + let current; + for (const line of gpg(['--with-colons', '--fixed-list-mode', '--list-keys', uid]).split('\n')) { + const fields = line.split(':'); + if (fields[0] === 'pub' || fields[0] === 'sub') current = fields[0]; + if (fields[0] === 'fpr') fprs.push({ kind: current, fpr: fields[9] }); + } + return { + primary: fprs.find((item) => item.kind === 'pub').fpr, + subkeys: fprs.filter((item) => item.kind === 'sub').map((item) => item.fpr), + }; +} + +/** @param {string} name @param {{past?: boolean, primary?: string, usage?: string}} [options] */ +function key(name, { past = false, primary = 'ed25519', usage = 'cert' } = {}) { + const uid = `${name} <${name}@fixture.invalid>`; + gpg(['--quick-gen-key', uid, primary, usage, past ? '1d' : 'never'], { past }); + return uid; +} + +/** @param {string} uid @param {string} algo @param {{past?: boolean, expire?: string}} [options] */ +function subkey(uid, algo, { past = false, expire = 'never' } = {}) { + gpg(['--quick-add-key', fingerprints(uid).primary, algo, 'encr', expire], { past }); +} + +/** Drop the AEAD feature, so encryption to this key uses SEIPDv1 with MDC (the approved profile). */ +function withoutAead(uid) { + gpg(['--command-fd', '0', '--edit-key', fingerprints(uid).primary], { + input: `setpref ${NO_AEAD}\ny\nsave\n`, + }); +} + +const fixtures = {}; +function record(name, uid, pin = (keys) => keys.subkeys[0]) { + const keys = fingerprints(uid); + writeFileSync(join(here, `${name}.asc`), gpg(['--armor', '--export', keys.primary])); + fixtures[name] = { primary: keys.primary, subkey: pin(keys) }; +} + +try { + let uid = key('good'); + subkey(uid, 'cv25519'); + withoutAead(uid); + record('good', uid); + + uid = key('other'); + subkey(uid, 'cv25519'); + withoutAead(uid); + record('other', uid); + + uid = key('good-aead'); + subkey(uid, 'cv25519'); + record('good-aead', uid); + + uid = key('rsa3072', { primary: 'rsa3072' }); + subkey(uid, 'rsa3072'); + withoutAead(uid); + record('rsa3072', uid); + + uid = key('rsa2048', { primary: 'rsa3072' }); + subkey(uid, 'rsa2048'); + withoutAead(uid); + record('rsa2048', uid); + + uid = key('expired-primary', { past: true }); + subkey(uid, 'cv25519', { past: true }); + record('expired-primary', uid); + + // A subkey cannot be added in the past to a key created now, so both are created in 2020. + uid = `expired-subkey `; + gpg(['--quick-gen-key', uid, 'ed25519', 'cert', 'never'], { past: true }); + subkey(uid, 'cv25519', { past: true, expire: '1d' }); + record('expired-subkey', uid); + + uid = key('revoked-primary'); + subkey(uid, 'cv25519'); + withoutAead(uid); + const revoked = fingerprints(uid).primary; + const certificate = readFileSync(join(home, 'openpgp-revocs.d', `${revoked}.rev`), 'utf8'); + gpg(['--import'], { input: certificate.replace(/^:-----BEGIN/mu, '-----BEGIN') }); + record('revoked-primary', uid); + + uid = key('revoked-subkey'); + subkey(uid, 'cv25519'); + withoutAead(uid); + gpg(['--command-fd', '0', '--edit-key', fingerprints(uid).primary], { + input: 'key 1\nrevkey\ny\n0\n\ny\nsave\n', + }); + record('revoked-subkey', uid); + + uid = key('sign-only', { usage: 'sign' }); + withoutAead(uid); + record('sign-only', uid, (keys) => keys.primary); + + uid = key('two-encryption-subkeys'); + subkey(uid, 'cv25519'); + subkey(uid, 'cv25519'); + withoutAead(uid); + record('two-encryption-subkeys', uid); + + writeFileSync(join(here, 'pins.json'), `${JSON.stringify(fixtures, null, 2)}\n`); + for (const name of readdirSync(here).filter((file) => file.endsWith('.asc'))) { + assert.ok(!readFileSync(join(here, name), 'utf8').includes('PRIVATE KEY'), name); + } +} finally { + spawnSync('gpgconf', ['--homedir', home, '--kill', 'all'], { stdio: 'ignore' }); + spawnSync('gpgconf', ['--homedir', home, '--remove-socketdir'], { stdio: 'ignore' }); + rmSync(home, { recursive: true, force: true }); +} diff --git a/tools/cli-package/custody-fixtures/good-aead.asc b/tools/cli-package/custody-fixtures/good-aead.asc new file mode 100644 index 00000000..26a01ecf --- /dev/null +++ b/tools/cli-package/custody-fixtures/good-aead.asc @@ -0,0 +1,14 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEat2WvhYJKwYBBAHaRw8BAQdAokSGPLhvXsbRjAjXgDsQIAV/PVVdF8wSQ/k2 +S2rlbxS0JWdvb2QtYWVhZCA8Z29vZC1hZWFkQGZpeHR1cmUuaW52YWxpZD6IrwQT +FgoAVxYhBEDvL/eVF4dpAtpA7QOQEPqAVWbWBQJq3Za+GxSAAAAAAAQADm1hbnUy +LDIuNSsxLjEyLDAsMwIbAQULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIXgAAKCRAD +kBD6gFVm1sCuAP45PAWVAvHneGtF1LkTKDgUcAG1ek59lsc3Ip0710fERwD+Ntae +o3P41B1X3bbBpjRfAUXzFqRg59R4qlOWagW4egS4OARq3Za+EgorBgEEAZdVAQUB +AQdAhOq0LdJOTD0hwThHIc95Nq8e0q6l7zU8C/keUEaz+kMDAQgHiJQEGBYKADwW +IQRA7y/3lReHaQLaQO0DkBD6gFVm1gUCat2WvhsUgAAAAAAEAA5tYW51MiwyLjUr +MS4xMiwwLDMCGwwACgkQA5AQ+oBVZtYOrQD/fEmoTIeq9y1uUSdOwAOBaCvsNwIc +Tp8pHASeA+zbK1sBAOLL1uDoEoHE8LCEOzI9rhoT1ygvTGuPRx92zueCBYoG +=AEJk +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tools/cli-package/custody-fixtures/good.asc b/tools/cli-package/custody-fixtures/good.asc new file mode 100644 index 00000000..a1b035f6 --- /dev/null +++ b/tools/cli-package/custody-fixtures/good.asc @@ -0,0 +1,14 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEat2WuxYJKwYBBAHaRw8BAQdAEeSA+thutMGa/RNuUGSIlsrNCdIlvuVfL7vU +0Zt/NhO0G2dvb2QgPGdvb2RAZml4dHVyZS5pbnZhbGlkPoipBBMWCgBRGxSAAAAA +AAQADm1hbnUyLDIuNSsxLjEyLDAsMwIbAQIXgBYhBO4GgcqIG3Sq5zqBtIO2uabd +MVVHBQJq3Za9BAsJCAcEFQoJCAQWAgEAAh4FAAoJEIO2uabdMVVHMwQBALApldn5 +z4MkMfYFH+MecSmW3VTLacxNbV9PePJksnnJAQCYYvTWWTZIavACiUzVznn8h3TI +xAWNMT3lOVRyz+dGAbg4BGrdlrwSCisGAQQBl1UBBQEBB0A1IbBf35QcZVaAsZsB +6dNA0Htnxozzm/aUH1zypb51WwMBCAeIlAQYFgoAPBYhBO4GgcqIG3Sq5zqBtIO2 +uabdMVVHBQJq3Za8GxSAAAAAAAQADm1hbnUyLDIuNSsxLjEyLDAsMwIbDAAKCRCD +trmm3TFVRwdIAP9vkzEvZbkbrrPq+3siG5Zfe+JGgGRFoSP/HkijhC3Z3gD+O98s +oAoEBcHwtAJHDRfBDg8t7uDk3IskjjlCGeFHiAA= +=csdU +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tools/cli-package/custody-fixtures/other.asc b/tools/cli-package/custody-fixtures/other.asc new file mode 100644 index 00000000..bc155cc8 --- /dev/null +++ b/tools/cli-package/custody-fixtures/other.asc @@ -0,0 +1,14 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEat2WvRYJKwYBBAHaRw8BAQdAAmv01jf91maMkpr2vIbVvWYF1fKi7dK6udWh +kZcNbjy0HW90aGVyIDxvdGhlckBmaXh0dXJlLmludmFsaWQ+iKkEExYKAFEbFIAA +AAAABAAObWFudTIsMi41KzEuMTIsMCwzAhsBAheAFiEEIUHtrrAwMYKW0f3tIQmz +3KSCa9UFAmrdlr4ECwkIBwQVCgkIBBYCAQACHgUACgkQIQmz3KSCa9U9VQEA1iuR +FMeHA75QQYyQrXsyFHyekW2w/h+zJfQ4WWieWEIA/jNp8tU/HRDqK+c2ix0yFn2l ++wjppRJa4Dcs4wJytX4BuDgEat2WvRIKKwYBBAGXVQEFAQEHQMnOIYQieOy4V1SF +p2nwvrqda7RFyFvJBzdiZiayCI1hAwEIB4iUBBgWCgA8FiEEIUHtrrAwMYKW0f3t +IQmz3KSCa9UFAmrdlr0bFIAAAAAABAAObWFudTIsMi41KzEuMTIsMCwzAhsMAAoJ +ECEJs9ykgmvV92AA/18cC4M9kz751q9WeRNEgLtoQ+2lPCk9Z7+OroRJnEpyAP4/ +++lEOf/ea4mqlBLzBsY24cPSbuN8Jp6seEbz/+lVBA== +=uHao +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tools/cli-package/custody-fixtures/pins.json b/tools/cli-package/custody-fixtures/pins.json new file mode 100644 index 00000000..ac167578 --- /dev/null +++ b/tools/cli-package/custody-fixtures/pins.json @@ -0,0 +1,46 @@ +{ + "good": { + "primary": "EE0681CA881B74AAE73A81B483B6B9A6DD315547", + "subkey": "F2975EB87A2A368097B3E2107DB14DAAA7F5853C" + }, + "other": { + "primary": "2141EDAEB030318296D1FDED2109B3DCA4826BD5", + "subkey": "0CB8BAB8E07E7FBECF4FB0698460B65F8FE5950C" + }, + "good-aead": { + "primary": "40EF2FF79517876902DA40ED039010FA805566D6", + "subkey": "378B6ADB6E8075395972FB73E06A4FB6DF7C698D" + }, + "rsa3072": { + "primary": "CB42858849CC31B9FD9E57D5CC6C94646B58DF14", + "subkey": "4F263156BC6D479C1155C6BFCABD5BC5AA2A354D" + }, + "rsa2048": { + "primary": "72ADEBD7526307F22475E4BF3E7BB55F4618AE3F", + "subkey": "9E1CB6EF6717695C78DD88EED4A281B5C0659AA8" + }, + "expired-primary": { + "primary": "E536026D5D94E4B15EE98400570DF90E46D872DD", + "subkey": "5063075D48B8F4A2807BA2E70BF054F449288B59" + }, + "expired-subkey": { + "primary": "EE99B28BE92E2C9ECFD050437B06E4A4A103F639", + "subkey": "E391FD0050FC3DFEC2BB9059E5C47BF3D6755E83" + }, + "revoked-primary": { + "primary": "948BC49F079C6015ECF03F93EE9216602EC57C79", + "subkey": "70517C46844F3641AC36ACDE97B2ECCEC091E981" + }, + "revoked-subkey": { + "primary": "C18A44CD2063D08ED2FE89490C0B99F8A32B2B99", + "subkey": "C65206D133BC917C7C74DEDD3497A20162C01EDB" + }, + "sign-only": { + "primary": "169B64C7A5E3F70681DE919275127ED14F938C6B", + "subkey": "169B64C7A5E3F70681DE919275127ED14F938C6B" + }, + "two-encryption-subkeys": { + "primary": "9DC1C32A1AD28FAA2961DA244718609DBBAAAAEC", + "subkey": "88F65CC8EFF54D6573EE9E6861C9EC83CE7526C2" + } +} diff --git a/tools/cli-package/custody-fixtures/revoked-primary.asc b/tools/cli-package/custody-fixtures/revoked-primary.asc new file mode 100644 index 00000000..0822fa2f --- /dev/null +++ b/tools/cli-package/custody-fixtures/revoked-primary.asc @@ -0,0 +1,18 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEat2WwBYJKwYBBAHaRw8BAQdAADIA3uUrKdTYtrCCJGABKLhAHBIJcCnwqwRC +/BfCJa6IjwQgFgoANxYhBJSLxJ8HnGAV7PA/k+6SFmAuxXx5BQJq3ZbAFiMElIvE +nwecYBXs8D+T7pIWYC7FfHkCHQAACgkQ7pIWYC7FfHmjEwEAuW3zR2FJylii03s+ +kk5ntI3TlTcALxptv4zPEbBICzsA/3pjoKqJkfiGFjbxyOjnNuMGNoOwepctwGRt +X4LXmfkAtDFyZXZva2VkLXByaW1hcnkgPHJldm9rZWQtcHJpbWFyeUBmaXh0dXJl +LmludmFsaWQ+iKkEExYKAFEbFIAAAAAABAAObWFudTIsMi41KzEuMTIsMCwzAhsB +AheAFiEElIvEnwecYBXs8D+T7pIWYC7FfHkFAmrdlsEECwkIBwQVCgkIBBYCAQAC +HgUACgkQ7pIWYC7FfHk65wEA1Kbs4+R2h5/aXQ2671g+zGnwsxPb2UbY3b+tlst3 +FjkBAOq+VYgnbxPUVRDmZy30TJJILVZ8m4Xd0tXV3paVZxwEuDgEat2WwBIKKwYB +BAGXVQEFAQEHQFVn2Nk0296KF3+1m68fkqzJKcJo1HvUosITb6mnHMd2AwEIB4iU +BBgWCgA8FiEElIvEnwecYBXs8D+T7pIWYC7FfHkFAmrdlsAbFIAAAAAABAAObWFu +dTIsMi41KzEuMTIsMCwzAhsMAAoJEO6SFmAuxXx5HnoA/R0dH75WZHtTveyk9TsV +lBG2b3wWOLVlqt4Rd/3oXTNQAP9F/fCqWN+eRlcMm0jSg6l5mV+hD8lyCBusDykg +eCrqDg== +=+COW +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tools/cli-package/custody-fixtures/revoked-subkey.asc b/tools/cli-package/custody-fixtures/revoked-subkey.asc new file mode 100644 index 00000000..c9c84283 --- /dev/null +++ b/tools/cli-package/custody-fixtures/revoked-subkey.asc @@ -0,0 +1,17 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEat2WwRYJKwYBBAHaRw8BAQdALhxOGRC0XdinhS/b3a+N50OgHtXop0hOr4Zf +i6Ru1ia0L3Jldm9rZWQtc3Via2V5IDxyZXZva2VkLXN1YmtleUBmaXh0dXJlLmlu +dmFsaWQ+iKkEExYKAFEbFIAAAAAABAAObWFudTIsMi41KzEuMTIsMCwzAhsBAheA +FiEEwYpEzSBj0I7S/olJDAuZ+KMrK5kFAmrdlsIECwkIBwQVCgkIBBYCAQACHgUA +CgkQDAuZ+KMrK5kRHQD/YfjCv+yB3RiDN049UQwMqfbis0367PKR2EC72iYUC/YB +AIkbNskO5fyUPdF/gkHQcHrAm0MtzxZ+xCUltCiWzEYBuDgEat2WwRIKKwYBBAGX +VQEFAQEHQL+g+kiOwZvQdRxFvB20Q4nBUh5hp6UBGmx1wbcN/7AgAwEIB4h4BCgW +CgAgFiEEwYpEzSBj0I7S/olJDAuZ+KMrK5kFAmrdlsMCHQAACgkQDAuZ+KMrK5ll +RgD/RVVXl0lSbq06DiRfa/vl+eMv7Uw8b/c3neReJyy6/XIA/AgcwY3t8LTaaa7c +NbQEWDitBJk51dZA/+tYmI+js18OiJQEGBYKADwWIQTBikTNIGPQjtL+iUkMC5n4 +oysrmQUCat2WwRsUgAAAAAAEAA5tYW51MiwyLjUrMS4xMiwwLDMCGwwACgkQDAuZ ++KMrK5l8lwD/ftWUHj6NO4yLPlqmtaUbJZyCBNXaDhzrs2lRzJjQChkA/2VaOnfV +uNHkWkRQLmpsTXKyzugnsRMxjCdcN8PjgvUB +=MNM3 +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tools/cli-package/custody-fixtures/rsa2048.asc b/tools/cli-package/custody-fixtures/rsa2048.asc new file mode 100644 index 00000000..7235b983 --- /dev/null +++ b/tools/cli-package/custody-fixtures/rsa2048.asc @@ -0,0 +1,39 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQGNBGrdlr8BDADeFkpg6V9UvTWp3T9MwSwqDi6mO71EcBXw8vAWr2WSibFEbdJg +nB2BVPV5GsT+ZaDJzaTjuhcxLcEjeZNHAFh6/6eKFWO4lQ9T1IPovH1gD4REZknp +p4QtcmxONjpFBLxKZt1tfSAWRM7q+fqqCRk+3nVk9HhAslqk+2/cBuVVaQppBYra +QVuh2f/Xs735oaBFw10K/fHnETWtByx1ugQgNzizhvfU7ST820V2oEimya0SQqcR +2WAUgYYBViCdz1j2SbCuXl0m69DaMXbOo3FX6xzCbGdKt9Ydr73g10ZO6zOXA/u/ +014ViO+jG8j8kR16X/FISg8dNGjj6ziB3iONYdloLWOy6yLwXX0Ebw6dgLozsnBa +P2CKw8LTdkQEHoNLxSM5Gb7e8h54w/VYDcdLBgd7jFA7YRfKg/IV5NyoJt9dSJOG +6fpedgk6IEqru7FEssBKHO3oWYrYSoBDOgqxm/VvsL8eEUdsmgFX5XUcR2Qfrsh3 +D9To9xcFTT/jxd0AEQEAAbQhcnNhMjA0OCA8cnNhMjA0OEBmaXh0dXJlLmludmFs +aWQ+iQHnBBMBCABRGxSAAAAAAAQADm1hbnUyLDIuNSsxLjEyLDAsMwIbAQIXgBYh +BHKt69dSYwfyJHXkvz57tV9GGK4/BQJq3ZbABAsJCAcEFQoJCAQWAgEAAh4FAAoJ +ED57tV9GGK4/fT0MAJiy3xH33wdTYWqEhXPiQ5H7U2vZi6EL/DiEF1TNJBtPuIFE +0j5hpHBiBragMvEbz+lpanMBY1639q54deFhWoNkImzw1klO6ANAAFBWbwsFob0w +Is4Mr0tO2INDVAjUyyrtLeF4igF+ybzCatxIn9Rr+liSJO/69ssjjvTaSB+JF/MX +S1dKvP5gHL6yEE8chPYTT9PHGr9k5GE55xW0ptP1tBesOjoHt91JjRW1HE3OEG96 +3h3rNbmk8Sy75SJ1owzaAYuhnFhzQusGg4PfahV7tfhHK255dM+GAUtGrSx4C2Q+ +L6KGaMNInFUQb1k4oBKn40Nso+z+np+5bjimVCC5pMVGHYvMKadq3yC+uZLmOlKo +G0LchhSrpXOuGroCrZ7C1u1BBzmIaV20dsqSCKGe/5d+EEMY3xoxYlerRQcUeIFZ +bQ9+CeC3REXAr0qP+7CUryfxzj7UJI4MkGY0lb8pmlirMk0dttANSDDycMeylAxj +3rLHUITtSSER05OxTLkBDQRq3ZbAAQgA6w7+ogqC8ws0ohCgrtrASoiwBqVHUENE +u7Va7v6w55WH/aqCcs3+FjmmOr7rUMdC3Lh3ktW4s7Y21yF452uzKuFO91ZKoinb +aRwBk/jXv3Wd11/KBC3Or2RD3NJAPhffTeCRcRWe5rg7fQ6r/Luk0VhI5dOeyDF7 +Us1eMFAPlolQrjGo9D67A4v+TTwEKp5LhIYMz30G0apK+y730nT/Fs0Bck1oVbFi +s0w6dG363S8a/k94jwY5csFI+9NR+YnGIIcHWm7113vJxpamzS6wI6OApGXBPWTR ++IPt3G1k04MwYxGDZCSSVOVWKIh1PJ9PIjVUriB5DQ1O4dubJNApRQARAQABiQHS +BBgBCAA8FiEEcq3r11JjB/IkdeS/Pnu1X0YYrj8FAmrdlsAbFIAAAAAABAAObWFu +dTIsMi41KzEuMTIsMCwzAhsMAAoJED57tV9GGK4/UHoL/3fKq0JDRmGl3g4ugABX +kDnjjuXg7YJIM+2SriHrnJKpqi/m+heoKtWWReo/QQAo4iEIK4+LR/9dmo29+FUv +B1XAAb4UBab4vn+mwsiqPkw+yIcgWK9SpNckt5zLU9Hh/HTvG/Is9yUb+caksfZM +De0LxmVu86YRgAPv+L0t3BjX9lqGsPSx6VD/z3FoZeBsYEnCMOuh1OjAyn/iRIh1 +pwJX/ySvd2ngACZBmT/4PwQTwJbnV9X68+JuIJpBKmaXGNurv/6tM00LUlOYuVtV +KanIuE85upkJvp0r8EeuCfhS/Ra757ZL6kaTQoiM9nIhjKflzI2s3ipTZwihaRI/ +dQm4FYjw8d7My6SDEiqwePYwHNcuZ7alXzcZVXufXV5zrrRY1Axuy5pXJ+jWS/Zd +JxjmdPFBW1PXv/R5clvzghB2fUHCI0oaP18zP8XusxYC2lXj5x3Ea024U4tZ4Vn8 +VgP0OaujPsdQEppwTvdDCtcLB6z2aWj3xTEdyPJF92s8lg== +=+VgZ +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tools/cli-package/custody-fixtures/rsa3072.asc b/tools/cli-package/custody-fixtures/rsa3072.asc new file mode 100644 index 00000000..145dcf7a --- /dev/null +++ b/tools/cli-package/custody-fixtures/rsa3072.asc @@ -0,0 +1,42 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQGNBGrdlr4BDACt3W1a3dBJebX1ZlnzR/PgP7MAJPi0Gpz4/ZEs4dl7t9/EjAVn +eJCaZGlfQtj2SwnmURvHWHPXTiLvQVaQcjwa6cfjf2/yIJsZcUcq4vPJjjV1IMGl +lZooVfITRVkBZdOlV/uXh7HBhjW9GQ/l2Jhe6HN4EK96vgk0jf9qWdEqqHjtXnSw +7PPYej2fxCny1kHRDFw5w8sAQmd4ZDmPhbm56G7rbxMqEcDkftdYIC4sarcn7b25 +7iXyDcgmuqyM1Mevi8RdbuCR/WVW6ZnqtylJ6q/iPXyagXFfwcJXWM8lJBU+5lBE +O1uCr+LVlJ/Vjlj1bLETq1rs/YdKd6IGZOgcfS8cOIsy66m7dcHZcdM1h52wiOMn +DTBWzIW6ionfWMtonThRrsSk0d/bM+U6+bCGf+V4jlby2+GLEkezfqP5+hngechA +SWJ4NuUUenwP9cjla12VVSr9KiA4IJoH/sdjUM2lFNXr+iJPqVy9nAxtq0LU8SUn +KKazdRyoD4f83/kAEQEAAbQhcnNhMzA3MiA8cnNhMzA3MkBmaXh0dXJlLmludmFs +aWQ+iQHnBBMBCABRGxSAAAAAAAQADm1hbnUyLDIuNSsxLjEyLDAsMwIbAQIXgBYh +BMtChYhJzDG5/Z5X1cxslGRrWN8UBQJq3Za/BAsJCAcEFQoJCAQWAgEAAh4FAAoJ +EMxslGRrWN8ULnAMAKQU+TngA84brOHZ/Y+8iXLHaIDpnKoBxCSXaKsWM8x7+g7J +PI5R2/Ok2uU1W9HI1genJjXwZnzExf2AoGzb0aXk/FvhjeXHT4sR1L1RNQJc2XAO +uyEf3/vWLDYWyJkS2TGt7d2eYmGWklrxsnvKVcGzc/mVCvY7dACjAditAJPmQ/rH +z41QYn1eKxDLqO2Y7+Dk8JuuisfQHHGCAOmriIGVmB+F+DWKatavx6BhtX6lGArX +n0iwpg1fTu0MpNLemQ6kXmEyVNJO51Gxp87Y5JABhCYxcerti45hB4gIkD3lx8v0 +zBdHnNvdCSzEsR247p4d+uaovvK4XSOUjFP04hPIaBwR/nsoqcinyhYStj7HOBL8 +iQ6ug85t/DWofUk/gEAmb1DCujb6VRSn8NOkDnebnCfQ8jBKW4flhFMPBaBCfviN +O1IrxQBKgvcSHWm2sbpAxy2GQMNFajVtCOOgHMq8ds77X+RDO845qfu0je+vZmkC +vEc7rS6QmMG87kZ89LkBjQRq3Za+AQwAxKgLJFYyz3n6GTymTdRlE3ZDYZgESUE4 +fYBP/APpJ2Nvo/UDAT7+5pgQ2aYba7Ovyzc03AJ0P+yGDttx3hFp25xZRz9n6qw1 +hUCWRDXCT1haYiYOWYuj8+CmuQN5IHJaFHsfF7foWinqPigS/P5Xe4yzGzoPPKpe +WPiA9kKLeUxqir5QKjNUmGIfaUIGrODbwbSST1pm/s842rM1xti6ghXr5pk7Uqq0 +8YzTUg6QJd2X95ssvvC7Ht4+viIRy9ldJnru5h1tuEtBAiU387Zqf+T7nDDKoNyQ +j0yY0IGJ8mBXuHJ4Xbjh87Esn0odhTgBqMan4jQg73Hg0AXlUT+daGEoWvKnVQrt +zDMyn90mMy8JZ9/LfZziNNOriM49+i6v1cf06KsbQkSZ9aXWnJOX+QtA8dfZymZh +h2qxj11mgkTICzDvdwKo1ngQJtn4VGNgYESOpo6kAIOfVig51mGf1MuHq2squo5L +BX2OuhDdTmRGC2ZTbbOaegMEy5wiWrvlABEBAAGJAdIEGAEIADwWIQTLQoWIScwx +uf2eV9XMbJRka1jfFAUCat2WvhsUgAAAAAAEAA5tYW51MiwyLjUrMS4xMiwwLDMC +GwwACgkQzGyUZGtY3xR3Mgv/QJo4zuU6c//uKPLA65zoH7WySjsl/iXwXUgCP7VJ +90LF9pbrfCmCz2uOSM4DhAlrFgtS0G43ZYN1yrfUENbZ1omrje8zPKUAAdsgE/Tq +A2phoxxvCsZF0d/QZtWIce9Til7w2tyBI3xMdg3nHXQ24LNgArQl6YY+MjuXAsIi +WJe4fycqEiyZtnAjkBvastl1D9RTUytqlTbHfjBuSQN2k6x//08FEzCP/qi2Vw1L +9Qi5+vOpOl1ukLtlYMlFMv9jNfn+z7kpOT/CEs9vlCH2LykORxmASurX/FhxmeyR +JkXOdzHR4rTVO/XLltWHHrHvQ1YV/+tLybkAXDFxBbE//Pk4ZBg/fczdkErRiTsJ +NbKGjgacDoUBAusWryN/egkJi+f73VN9YlKTCGueNHM5OEol2i49eTbTQmUeE2Is +G8F+yIY0MvPBvICfLfmgbUlp0hNYnDphQptuwgkXJ8yQlGbONuq8HHcxB/aDoR6R +MZTE5tsCyD89p/Wr9u6XU+Ks +=GO/z +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tools/cli-package/custody-fixtures/sign-only.asc b/tools/cli-package/custody-fixtures/sign-only.asc new file mode 100644 index 00000000..435141b5 --- /dev/null +++ b/tools/cli-package/custody-fixtures/sign-only.asc @@ -0,0 +1,10 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEat2WwxYJKwYBBAHaRw8BAQdAGfsw2NVWKZsXoJ3mayMQoqGRmDGPXWosOe57 +8OCGEQq0JXNpZ24tb25seSA8c2lnbi1vbmx5QGZpeHR1cmUuaW52YWxpZD6IqQQT +FgoAURsUgAAAAAAEAA5tYW51MiwyLjUrMS4xMiwwLDMCGwMCF4AWIQQWm2THpeP3 +BoHekZJ1En7RT5OMawUCat2WxAQLCQgHBBUKCQgEFgIBAAIeBQAKCRB1En7RT5OM +ax7QAQDWs8vQhT20VQif75MxOKuuJ+5F0qhOJasSYNZmpda1ywD/QiAP8TRB+E8X +x9X9DnNPEngLC8u1dBhtuq0pxLkN9Ag= +=K6zy +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tools/cli-package/custody-fixtures/two-encryption-subkeys.asc b/tools/cli-package/custody-fixtures/two-encryption-subkeys.asc new file mode 100644 index 00000000..938105a4 --- /dev/null +++ b/tools/cli-package/custody-fixtures/two-encryption-subkeys.asc @@ -0,0 +1,19 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEat2WxBYJKwYBBAHaRw8BAQdAj9pjkGVc96gIzspDm/Avvhz4Gmw/R05P7h5F +5KP3Jcm0P3R3by1lbmNyeXB0aW9uLXN1YmtleXMgPHR3by1lbmNyeXB0aW9uLXN1 +YmtleXNAZml4dHVyZS5pbnZhbGlkPoipBBMWCgBRGxSAAAAAAAQADm1hbnUyLDIu +NSsxLjEyLDAsMwIbAQIXgBYhBJ3Bwyoa0o+qKWHaJEcYYJ27qqrsBQJq3ZbFBAsJ +CAcEFQoJCAQWAgEAAh4FAAoJEEcYYJ27qqrsDv4BALGZRcIh5gxL8xBuqDW8Cfrv +fMJjKLqEvQLkrLYPqKv8AQC8F5+GeKVO5lP4AQyUq8IcPk/mXoH/zut6W8iRVSXj +BLg4BGrdlsQSCisGAQQBl1UBBQEBB0BF5owCYus+Zt9BlsRveQcp9ajIl/XjGe18 +45wVSTvRdQMBCAeIlAQYFgoAPBYhBJ3Bwyoa0o+qKWHaJEcYYJ27qqrsBQJq3ZbE +GxSAAAAAAAQADm1hbnUyLDIuNSsxLjEyLDAsMwIbDAAKCRBHGGCdu6qq7H44AP9e +OmONEnukZGfF4SpwqVEA2pklQIIf4aV2pqIvZEM2UgD+O3x+FqlA+pf/9+Av4o75 +M146LG+aglO5eeEDKDBDWQy4OARq3ZbEEgorBgEEAZdVAQUBAQdAL0hGquRkWGKp +lwNnMT+knhxxQ509QnwOL90d6yLOrE4DAQgHiJQEGBYKADwWIQSdwcMqGtKPqilh +2iRHGGCdu6qq7AUCat2WxBsUgAAAAAAEAA5tYW51MiwyLjUrMS4xMiwwLDMCGwwA +CgkQRxhgnbuqquwfFwD/SQDfpajkcPFt77Lcvg9WEnNP0MJXLDjB+4GX+EDY62gA +/2TjAiKPad4XtWJ92oJM46C4GF49XxBtB2UzTOlLYZgN +=+Sh7 +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tools/cli-package/gpg-fixture.mjs b/tools/cli-package/gpg-fixture.mjs new file mode 100644 index 00000000..ca8f7c9a --- /dev/null +++ b/tools/cli-package/gpg-fixture.mjs @@ -0,0 +1,100 @@ +/** + * Test-only GnuPG fixtures (Step 9b.1): the committed synthetic public keys and their pins, and an + * ephemeral decryption key generated per run in a short owner-only home. That home's agent is stopped + * and its socket directory removed when the test ends; no secret key is written anywhere else, and + * none is ever committed. + */ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { after } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { resolveGpg } from './gnupg.mjs'; + +export const FIXTURES = fileURLToPath(new URL('./custody-fixtures/', import.meta.url)); +/** @type {Record} */ +export const PINS = JSON.parse(readFileSync(join(FIXTURES, 'pins.json'), 'utf8')); +/** @param {string} name */ +export const fixture = (name) => join(FIXTURES, `${name}.asc`); + +const NO_AEAD = 'AES256 AES192 AES SHA512 SHA384 SHA256 ZLIB ZIP Uncompressed'; +/** The longest agent socket name GnuPG creates in a home, against macOS's 104-byte socket path limit. */ +const SOCKET = '/S.gpg-agent.browser'; + +/** Every process whose command line names `home`. */ +export function processesNaming(home) { + const listing = spawnSync( + 'ps', + process.platform === 'darwin' ? ['-axo', 'command'] : ['-eo', 'args'], + { encoding: 'utf8' }, + ); + return listing.stdout.split('\n').filter((line) => line.includes(home)); +} + +/** + * A fresh synthetic key pair whose secret key stays in its own short home: the public key (only the + * pinned encryption subkey) in a file, its pins, the secret key's export for refusal tests, and + * decryption. The home's agent is stopped and the home removed after the test file. + */ +export function ephemeralKey() { + const gpg = resolveGpg(); + const home = mkdtempSync(join(tmpdir(), 'rg-')); + assert.ok(home.length + SOCKET.length < 100, 'the ephemeral key home path is too long'); + const exportDirectory = mkdtempSync(join(tmpdir(), 'rge-')); + const env = { PATH: process.env.PATH ?? '', LANG: 'C', LC_ALL: 'C' }; + /** @param {string[]} args @param {string} [input] */ + const call = (args, input) => { + const run = spawnSync( + gpg.command, + [ + ...gpg.prefix, + '--homedir', + home, + '--batch', + '--no-tty', + '--no-options', + '--pinentry-mode', + 'loopback', + '--passphrase', + '', + ...args, + ], + { encoding: 'utf8', input, env, timeout: 60_000 }, + ); + assert.equal(run.status, 0, `ephemeral gpg ${args[0]} failed`); + return run.stdout; + }; + const dispose = () => { + spawnSync('gpgconf', ['--homedir', home, '--kill', 'all'], { stdio: 'ignore', env }); + spawnSync('gpgconf', ['--homedir', home, '--remove-socketdir'], { stdio: 'ignore', env }); + rmSync(home, { recursive: true, force: true }); + rmSync(exportDirectory, { recursive: true, force: true }); + }; + after(dispose); + const uid = 'ephemeral '; + call(['--quick-gen-key', uid, 'ed25519', 'cert', 'never']); + const listing = () => + call(['--with-colons', '--fixed-list-mode', '--with-subkey-fingerprint', '--list-keys', uid]) + .split('\n') + .filter((line) => line.startsWith('fpr:')) + .map((line) => line.split(':')[9]); + const [primary] = listing(); + call(['--quick-add-key', primary, 'cv25519', 'encr', 'never']); + call(['--command-fd', '0', '--edit-key', primary], `setpref ${NO_AEAD}\ny\nsave\n`); + const [, subkey] = listing(); + const file = join(exportDirectory, 'public.asc'); + writeFileSync(file, call(['--armor', '--export', `${subkey}!`])); + const secret = join(exportDirectory, 'secret.asc'); + writeFileSync(secret, call(['--armor', '--export-secret-keys', primary])); + return { + home, + file, + secret, + pins: { primary, subkey }, + /** @param {string} ciphertext @param {string} output */ + decrypt: (ciphertext, output) => call(['--output', output, '--decrypt', ciphertext]), + dispose, + }; +} From 85a9aebe81eac1c93cb5fccfcb4c974b094854d2 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 08:56:05 +0300 Subject: [PATCH 68/97] feat(repo): encrypt a private container once to one pinned recipient encryptOnce takes a container that reads back under the container checks, a recipient file with maintainer-confirmed pins (40-hex primary and subkey fingerprints), a destination and the caller's owner-only private root. A colon listing classifies the recipient first, in a fixed order: secret material, a missing pinned primary, more than one key, a missing pinned subkey, revocation, expiry, unusability (validity, disabled, capability, future creation, or an encryption subkey other than cv25519 or RSA >= 3072) and another usable encryption subkey; an armored secret key refuses from its bytes. GnuPG's own refusal stays a second layer. Success requires exit 0, one KEY_CONSIDERED for the pinned primary, one BEGIN_ENCRYPTION 2 9 (SEIPDv1 with MDC, AES-256; AEAD refuses) before one END_ENCRYPTION, exactly the expected work files, a ciphertext within its bound naming exactly the pinned subkey, and homes holding only what GnuPG creates. An existing ciphertext refuses before any GnuPG call and the publish never overwrites. Plaintext lives in an owner-only work directory removed on every path; an interrupt in the caller's scope stops GnuPG and removes the root. Errors are fixed PACKED_CLI_CUSTODY_* codes. Tests use real GnuPG with the committed fixtures, an ephemeral round trip decrypted and extracted by the platform tar, injected status, recipient, asset and home faults, placement, diagnostics and an interrupt harness. Refs: ADR-0111, Step 9b.1 Co-Authored-By: Claude Opus 5.5 --- package.json | 2 +- tools/cli-package/custody.mjs | 316 ++++++++++++++ tools/cli-package/custody.test.mjs | 654 +++++++++++++++++++++++++++++ 3 files changed, 971 insertions(+), 1 deletion(-) create mode 100644 tools/cli-package/custody.mjs create mode 100644 tools/cli-package/custody.test.mjs diff --git a/package.json b/package.json index bfdd15db..1f9cee7e 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "smoke:budget-replay": "node tools/budget-replay-compat/check.mjs", "check:wire-mutations": "node tools/wire-conformance/check.mjs", "check:usage-mutations": "node tools/usage-coverage-controls/check.mjs", - "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/ustar.test.mjs tools/cli-package/container.test.mjs tools/cli-package/gnupg.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/check.test.mjs tools/cli-package/paths.test.mjs tools/cli-package/source-map.test.mjs tools/cli-package/manifest.test.mjs tools/cli-package/symbolicate.test.mjs tools/cli-package/map-controls.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", + "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/ustar.test.mjs tools/cli-package/container.test.mjs tools/cli-package/gnupg.test.mjs tools/cli-package/custody.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/check.test.mjs tools/cli-package/paths.test.mjs tools/cli-package/source-map.test.mjs tools/cli-package/manifest.test.mjs tools/cli-package/symbolicate.test.mjs tools/cli-package/map-controls.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", "check:fs-mutations": "node tools/fs-policy-controls/check.mjs", "check:secrets-mutations": "node tools/secrets-controls/check.mjs", "smoke:cli-bootstrap": "node tools/cli-bootstrap/check.mjs", diff --git a/tools/cli-package/custody.mjs b/tools/cli-package/custody.mjs new file mode 100644 index 00000000..226bf716 --- /dev/null +++ b/tools/cli-package/custody.mjs @@ -0,0 +1,316 @@ +/** + * Recipient custody (Step 9b.1; ADR-0111; packaging.md, "Recipient custody"): encrypt a private + * container once to one pinned recipient. The recipient file must hold exactly one OpenPGP v4 key + * whose only usable encryption subkey is the pinned one, with a cv25519 or RSA ≥ 3072 encryption + * subkey, neither expired, revoked nor disabled; a colon listing classifies it before GnuPG's own + * refusal, a second, independent layer. The ciphertext must be SEIPDv1 with MDC and AES-256, carry + * exactly one recipient — the pinned subkey — and is published once, never over an existing file. + * Plaintext lives only in an owner-only work directory under the caller's private root, removed on + * every path; the caller's interrupt scope stops GnuPG and removes that root. This library holds no + * production recipient: Step 9b uses synthetic recipients only. + */ +import { lstatSync, mkdtempSync, readdirSync, realpathSync, rmSync } from 'node:fs'; +import { isAbsolute, join, relative, sep } from 'node:path'; +import { isDeepStrictEqual } from 'node:util'; +import { readPrivateContainer } from './container.mjs'; +import { CustodyError, refuseCustody } from './custody-error.mjs'; +import { readOrdinaryFile, sha256 } from './files.mjs'; +import { writePrivateFile } from './finalize.mjs'; +import { + GNUPG_CALL, + assertHomeState, + createGnupgHome, + encryptArgs, + gnupgEnvironment, + gnupgVersion, + listArgs, + parseColons, + parseStatus, + recipientsArgs, + resolveGpg, +} from './gnupg.mjs'; +import { runCommand } from './process.mjs'; + +const FINGERPRINT = /^[0-9A-F]{40}$/u; +/** An armored secret key: refused from the bytes, since listing one needs an agent this boundary never starts. */ +const SECRET_ARMOR = ['-----BEGIN PGP', 'PRIVATE KEY BLOCK-----'].join(' '); +const RECIPIENT_BYTES = 256 * 1024; +/** Literal-packet and packet framing overhead a ciphertext may add to its container. */ +const CIPHERTEXT_OVERHEAD = 64 * 1024; +/** The keywords a successful encryption never reports. */ +const FAILURE_STATUS = Object.freeze([ + 'INV_RECP', + 'NO_RECP', + 'FAILURE', + 'ERROR', + 'KEYEXPIRED', + 'KEYREVOKED', + 'INV_SGNR', +]); + +/** The ciphertext's file name for a package version. */ +export const ciphertextName = (version) => `relavium-${version}-private-maps.tar.gpg`; + +/** + * Maintainer-confirmed pins: the primary and encryption-subkey fingerprints, 40 uppercase hex digits + * each (OpenPGP v4). A user ID, a short key ID or a `!` suffix refuses. + * @param {unknown} pins + * @returns {{primary: string, subkey: string}} + */ +export function parsePins(pins) { + if ( + pins === null || + typeof pins !== 'object' || + !isDeepStrictEqual(Object.keys(pins).sort(), ['primary', 'subkey']) || + !FINGERPRINT.test(String(/** @type {Record} */ (pins).primary)) || + !FINGERPRINT.test(String(/** @type {Record} */ (pins).subkey)) + ) { + refuseCustody('PIN_FORMAT'); + } + const { primary, subkey } = /** @type {{primary: string, subkey: string}} */ (pins); + return Object.freeze({ primary, subkey }); +} + +/** + * The keys of a colon listing, each with its fingerprint. + * @param {string[][]} records + */ +function keysOf(records) { + /** @type {{type: string, fields: string[], fpr?: string}[]} */ + const keys = []; + for (const fields of records) { + if (['pub', 'sec', 'sub', 'ssb'].includes(fields[0])) keys.push({ type: fields[0], fields }); + else if (fields[0] === 'fpr' && keys.length > 0 && keys.at(-1).fpr === undefined) { + keys.at(-1).fpr = fields[9]; + } + } + return keys; +} + +/** @param {string[]} fields @param {number} now */ +const expired = (fields, now) => + fields[1] === 'e' || (fields[6] !== '' && Number(fields[6]) <= now); + +/** @param {string[]} fields @param {number} now */ +const usableEncryption = (fields, now) => + !['e', 'r', 'i', 'n', 'd'].includes(fields[1]) && + !expired(fields, now) && + fields[11].includes('e') && + !fields[11].includes('D'); + +/** The approved encryption-subkey algorithms: cv25519 ECDH, or RSA of at least 3072 bits. */ +const approvedAlgorithm = (fields) => + (fields[3] === '18' && fields[16] === 'cv25519') || + (fields[3] === '1' && Number(fields[2]) >= 3072); + +/** + * Classify a recipient listing against its pins, in a fixed order: secret material, a missing primary, + * more than one key, a missing subkey, revocation, expiry, unusability, then another usable + * encryption subkey. + * @param {string[][]} records colon records of `--show-keys` + * @param {{primary: string, subkey: string}} pins + * @param {number} now seconds since the epoch + */ +export function inspectRecipient(records, pins, now) { + const keys = keysOf(records); + if (keys.some((key) => key.type === 'sec' || key.type === 'ssb')) { + refuseCustody('RECIPIENT_SECRET'); + } + const primaries = keys.filter((key) => key.type === 'pub'); + const primary = primaries.find((key) => key.fpr === pins.primary); + if (!primary) refuseCustody('RECIPIENT_MISSING'); + if (primaries.length !== 1) refuseCustody('RECIPIENT_UNEXPECTED'); + const subkeys = keys.filter((key) => key.type === 'sub'); + const pinned = keys.find((key) => key.fpr === pins.subkey); + if (!pinned) refuseCustody('RECIPIENT_MISSING'); + if (primary.fields[1] === 'r' || pinned.fields[1] === 'r') refuseCustody('RECIPIENT_REVOKED'); + if (expired(primary.fields, now) || expired(pinned.fields, now)) { + refuseCustody('RECIPIENT_EXPIRED'); + } + if ( + pinned.type !== 'sub' || + ['i', 'n', 'd'].includes(primary.fields[1]) || + primary.fields[11].includes('D') || + !primary.fields[11].includes('E') || + !usableEncryption(pinned.fields, now) || + Number(primary.fields[5]) > now || + Number(pinned.fields[5]) > now || + !approvedAlgorithm(pinned.fields) + ) { + refuseCustody('RECIPIENT_UNUSABLE'); + } + if (subkeys.some((key) => key !== pinned && usableEncryption(key.fields, now))) { + refuseCustody('RECIPIENT_UNEXPECTED'); + } + return pins; +} + +/** @param {string} root @param {string} path */ +function within(root, path) { + const rel = relative(root, path); + return rel === '' || (!isAbsolute(rel) && rel !== '..' && !rel.startsWith(`..${sep}`)); +} + +/** An owner-only directory of this user, not a link. */ +function privateDirectory(path) { + const info = lstatSync(path); + return ( + info.isDirectory() && + !info.isSymbolicLink() && + (process.platform === 'win32' || + ((info.mode & 0o077) === 0 && + (typeof process.getuid !== 'function' || info.uid === process.getuid()))) + ); +} + +/** + * @typedef {{name: string, bytes: number, sha256: string, + * recipient: {primary: string, subkey: string}, gnupg: string}} CustodyRecord + */ + +/** + * Encrypt one container once to one pinned recipient and publish the ciphertext into `destination`. + * @param {{ + * container: Buffer, + * recipient: {file: string, pins: unknown}, + * destination: string, + * privateRoot: string, + * now?: number, + * run?: typeof runCommand, + * gpg?: {command: string, prefix: string[]}, + * }} options + * @returns {Promise} + */ +export async function encryptOnce({ + container, + recipient, + destination, + privateRoot, + now = Math.floor(Date.now() / 1000), + run = runCommand, + gpg, +}) { + const pins = parsePins(recipient.pins); + const { manifest } = readPrivateContainer(container); + const name = ciphertextName(manifest.packageVersion); + const destinationInfo = lstatSync(destination); + if (!destinationInfo.isDirectory() || destinationInfo.isSymbolicLink()) { + refuseCustody('PLACEMENT'); + } + if (readdirSync(destination).includes(name)) refuseCustody('ALREADY_ENCRYPTED'); + if (!privateDirectory(privateRoot)) refuseCustody('PRIVATE_ROOT'); + const destinationPath = realpathSync(destination); + const root = realpathSync(privateRoot); + if (within(root, destinationPath) || within(destinationPath, root)) refuseCustody('PLACEMENT'); + const binary = gpg ?? resolveGpg(); + + const work = mkdtempSync(join(root, 'custody-')); + try { + const env = gnupgEnvironment(work); + const listing = createGnupgHome(work, 'listing'); + const encryption = createGnupgHome(work, 'encryption'); + const recipientsHome = createGnupgHome(work, 'recipients'); + const call = async (args, code) => { + try { + return await run(binary.command, [...binary.prefix, ...args], { ...GNUPG_CALL, env }); + } catch { + return refuseCustody(code); + } + }; + const version = await gnupgVersion(binary, { run, env, home: encryption }); + + // Every later step reads this copy, so the bytes checked are the bytes encrypted to. + let key; + try { + key = readOrdinaryFile(recipient.file, RECIPIENT_BYTES); + } catch { + refuseCustody('RECIPIENT_UNREADABLE'); + } + if (key.includes(SECRET_ARMOR)) refuseCustody('RECIPIENT_SECRET'); + const recipientPath = join(work, 'recipient.asc'); + writePrivateFile(recipientPath, key); + const listed = await call(listArgs(listing, recipientPath), 'RECIPIENT_UNREADABLE'); + inspectRecipient(parseColons(String(listed.stdout)), pins, now); + assertHomeState(listing, ['trustdb.gpg']); + + const input = join(work, 'container.tar'); + const output = join(work, 'ciphertext.gpg'); + writePrivateFile(input, container); + const encrypted = await call( + encryptArgs(encryption, recipientPath, input, output), + 'ENCRYPTION_FAILED', + ); + const status = parseStatus(String(encrypted.stdout)); + if (status.some((line) => FAILURE_STATUS.includes(line.keyword))) refuseCustody('STATUS'); + const begins = status.filter((line) => line.keyword === 'BEGIN_ENCRYPTION'); + const ends = status.filter((line) => line.keyword === 'END_ENCRYPTION'); + const considered = status.filter((line) => line.keyword === 'KEY_CONSIDERED'); + if ( + begins.length !== 1 || + ends.length !== 1 || + status.indexOf(ends[0]) < status.indexOf(begins[0]) || + considered.length !== 1 || + !isDeepStrictEqual(considered[0].args.slice(0, 2), [pins.primary, '0']) + ) { + refuseCustody('STATUS'); + } + // SEIPDv1 with MDC (method 2) and AES-256 (algorithm 9); an AEAD algorithm refuses. + if (begins[0].args[0] !== '2' || begins[0].args[1] !== '9' || begins[0].args.length > 2) { + refuseCustody('PROFILE'); + } + assertHomeState(encryption, ['random_seed']); + + if ( + !isDeepStrictEqual(readdirSync(work).sort(), [ + 'ciphertext.gpg', + 'container.tar', + 'encryption', + 'listing', + 'recipient.asc', + 'recipients', + ]) + ) { + refuseCustody('ASSET'); + } + let ciphertext; + try { + ciphertext = readOrdinaryFile(output, container.length + CIPHERTEXT_OVERHEAD); + } catch { + refuseCustody('ASSET'); + } + const listedRecipients = await call(recipientsArgs(recipientsHome, output), 'RECIPIENTS'); + const recipients = parseStatus(String(listedRecipients.stdout)).filter( + (line) => line.keyword === 'ENC_TO', + ); + if ( + recipients.length !== 1 || + recipients[0].args[0] !== pins.subkey.slice(-16) || + recipients[0].args[0] === '0'.repeat(16) + ) { + refuseCustody('RECIPIENTS'); + } + assertHomeState(recipientsHome, []); + + try { + writePrivateFile(join(destinationPath, name), ciphertext); + } catch (error) { + if (error instanceof Error && 'code' in error && error.code === 'EEXIST') { + refuseCustody('ALREADY_ENCRYPTED'); + } + throw error; + } + return Object.freeze({ + name, + bytes: ciphertext.length, + sha256: sha256(ciphertext), + recipient: Object.freeze({ primary: pins.primary, subkey: pins.subkey }), + gnupg: version, + }); + } catch (error) { + // A host error's message can name a private path: only fixed custody codes leave. + if (error instanceof CustodyError) throw error; + throw new CustodyError('UNEXPECTED'); + } finally { + rmSync(work, { recursive: true, force: true }); + } +} diff --git a/tools/cli-package/custody.test.mjs b/tools/cli-package/custody.test.mjs new file mode 100644 index 00000000..9ee3f4e1 --- /dev/null +++ b/tools/cli-package/custody.test.mjs @@ -0,0 +1,654 @@ +import assert from 'node:assert/strict'; +import { spawn, spawnSync } from 'node:child_process'; +import { Buffer } from 'node:buffer'; +import { + appendFileSync, + chmodSync, + existsSync, + mkdirSync, + readFileSync, + readdirSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { dirname, join } from 'node:path'; +import { before, test } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { candidate, fresh, posixModes, seal } from './candidate-fixture.mjs'; +import { buildPrivateContainer } from './container.mjs'; +import { CustodyError } from './custody-error.mjs'; +import { ciphertextName, encryptOnce, inspectRecipient, parsePins } from './custody.mjs'; +import { createGnupgHome, gnupgEnvironment, listArgs, parseColons, resolveGpg } from './gnupg.mjs'; +import { PINS, ephemeralKey, fixture, processesNaming } from './gpg-fixture.mjs'; +import { readPrivateManifest } from './manifest.mjs'; +import { runCommand } from './process.mjs'; + +const refuses = (code) => (error) => error instanceof CustodyError && error.code === code; +const NOW = Math.floor(Date.now() / 1000); + +/** @type {Buffer} */ +let container; +/** @type {string} */ +let staging; +before(async () => { + // A synthetic candidate sealed as the producer seals it: synthetic maps only, never a product's. + const fixtureCandidate = await candidate(); + await seal(fixtureCandidate); + writeFileSync(join(fixtureCandidate.staging, 'metafile-esm.json'), '{"inputs":{},"outputs":{}}'); + staging = fixtureCandidate.staging; + container = buildPrivateContainer(staging).bytes; +}); +const NAME = () => ciphertextName(readPrivateManifest(staging).packageVersion); + +/** A fresh owner-only private root and an empty destination. */ +function places() { + const root = fresh('custody-root'); + mkdirSync(root); + chmodSync(root, 0o700); + const destination = fresh('custody-destination'); + mkdirSync(destination); + return { root, destination }; +} + +/** @param {string[]} args */ +const kindOf = (args) => + args.includes('--version') + ? 'version' + : args.includes('--show-keys') + ? 'list' + : args.includes('--encrypt') + ? 'encrypt' + : args.includes('--list-only') + ? 'recipients' + : 'other'; +/** @param {string[]} args @param {string} flag */ +const valueOf = (args, flag) => args[args.indexOf(flag) + 1]; + +/** + * A run port that records every GnuPG call and lets a test act before one, replace it, or rewrite + * its result; everything else is the real GnuPG. + * @param {Record} [hooks] + */ +function recording(hooks = {}) { + const calls = []; + /** @type {typeof runCommand} */ + const run = async (command, args, options) => { + const kind = kindOf(args); + calls.push(kind); + const hook = hooks[kind] ?? {}; + if (hook.before) await hook.before(args); + if (hook.replace) return hook.replace(args); + const result = await runCommand(command, args, options); + return hook.after ? hook.after(result, args) : result; + }; + return { run, calls }; +} + +/** One encryption attempt with fresh places; good recipient unless overridden. */ +async function attempt(overrides = {}) { + const { root, destination } = places(); + try { + const record = await encryptOnce({ + container, + recipient: { file: fixture('good'), pins: PINS.good }, + destination, + privateRoot: root, + ...overrides, + }); + return { root, destination, record }; + } catch (error) { + return { root, destination, error }; + } +} + +/** A refusal with its code that removed its plaintext work directory and published nothing. */ +function refused(outcome, code) { + assert.ok(outcome.error instanceof CustodyError, `expected ${code}, got ${outcome.error}`); + assert.equal(outcome.error.code, code); + assert.deepEqual(readdirSync(outcome.root), [], 'plaintext work directory left'); + assert.deepEqual(readdirSync(outcome.destination), [], 'something was published'); +} + +/** A colon key record with the shape of a good v4 key unless overridden. */ +function key(type, overrides = {}) { + const fields = Array(20).fill(''); + fields[0] = type; + fields[1] = '-'; + fields[2] = type === 'pub' ? '255' : '255'; + fields[3] = type === 'pub' ? '22' : '18'; + fields[4] = 'KEYID'; + fields[5] = String(NOW - 86_400); + fields[8] = '-'; + fields[11] = type === 'pub' ? 'cEC' : 'e'; + fields[16] = type === 'pub' ? 'ed25519' : 'cv25519'; + for (const [index, value] of Object.entries(overrides)) fields[Number(index)] = value; + return fields; +} +/** @param {string} value */ +const fpr = (value) => Object.assign(Array(11).fill(''), { 0: 'fpr', 9: value }); +const PRIMARY = 'A'.repeat(40); +const SUBKEY = 'B'.repeat(40); +/** A good key's records with the given field overrides on its primary and subkey. */ +const records = (pub = {}, sub = {}) => [ + key('pub', pub), + fpr(PRIMARY), + key('sub', sub), + fpr(SUBKEY), +]; + +test('pins are exactly two 40-digit uppercase fingerprints', () => { + assert.deepEqual(parsePins({ primary: PRIMARY, subkey: SUBKEY }), { + primary: PRIMARY, + subkey: SUBKEY, + }); + for (const pins of [ + undefined, + { primary: PRIMARY }, + { primary: PRIMARY, subkey: SUBKEY, extra: SUBKEY }, + { primary: PRIMARY.toLowerCase(), subkey: SUBKEY }, + { primary: PRIMARY, subkey: `${SUBKEY}!` }, + { primary: PRIMARY, subkey: SUBKEY.slice(-16) }, + { primary: 'good ', subkey: SUBKEY }, + { primary: 'C'.repeat(64), subkey: SUBKEY }, + ]) { + assert.throws(() => parsePins(pins), refuses('PIN_FORMAT'), JSON.stringify(pins)); + } +}); + +test('the recipient classification refuses in its fixed order', () => { + const pins = { primary: PRIMARY, subkey: SUBKEY }; + assert.equal(inspectRecipient(records(), pins, NOW), pins); + for (const [code, listing, override] of [ + ['RECIPIENT_SECRET', [key('sec'), fpr(PRIMARY), key('ssb'), fpr(SUBKEY)]], + ['RECIPIENT_MISSING', []], + ['RECIPIENT_MISSING', records(), { primary: 'C'.repeat(40), subkey: SUBKEY }], + ['RECIPIENT_MISSING', records(), { primary: PRIMARY, subkey: 'C'.repeat(40) }], + ['RECIPIENT_UNEXPECTED', [...records(), key('pub'), fpr('D'.repeat(40))]], + ['RECIPIENT_REVOKED', records({ 1: 'r' })], + ['RECIPIENT_REVOKED', records({}, { 1: 'r' })], + ['RECIPIENT_EXPIRED', records({ 1: 'e' })], + ['RECIPIENT_EXPIRED', records({}, { 6: String(NOW) })], + ['RECIPIENT_UNUSABLE', records({ 1: 'i' })], + ['RECIPIENT_UNUSABLE', records({ 11: 'cECD' })], + ['RECIPIENT_UNUSABLE', records({ 11: 'cC' })], + ['RECIPIENT_UNUSABLE', records({}, { 11: 's' })], + ['RECIPIENT_UNUSABLE', records({}, { 1: 'n' })], + ['RECIPIENT_UNUSABLE', records({ 5: String(NOW + 60) })], + ['RECIPIENT_UNUSABLE', records({}, { 5: String(NOW + 60) })], + ['RECIPIENT_UNUSABLE', records({}, { 16: 'nistp256' })], + ['RECIPIENT_UNUSABLE', records({}, { 3: '1', 2: '2048', 16: '' })], + ['RECIPIENT_UNUSABLE', records(), { primary: PRIMARY, subkey: PRIMARY }], + ['RECIPIENT_UNEXPECTED', [...records(), key('sub'), fpr('E'.repeat(40))]], + ]) { + assert.throws( + () => inspectRecipient(listing, override ?? pins, NOW), + refuses(code), + `${code} ${JSON.stringify(listing.map((fields) => fields.slice(0, 12)))}`, + ); + } + // RSA of at least 3072 bits is approved; an expired or revoked second subkey is not unexpected. + assert.equal(inspectRecipient(records({}, { 3: '1', 2: '3072', 16: '' }), pins, NOW), pins); + assert.equal( + inspectRecipient([...records(), key('sub', { 1: 'r' }), fpr('E'.repeat(40))], pins, NOW), + pins, + ); +}); + +test('the committed synthetic fixtures list as their names say under the real GnuPG', async () => { + const gpg = resolveGpg(); + const work = fresh('fixture-listing'); + mkdirSync(work); + let count = 0; + const classify = async (file, pins) => { + const home = createGnupgHome(work, `home-${(count += 1)}`); + const listed = await runCommand(gpg.command, [...gpg.prefix, ...listArgs(home, file)], { + timeoutMs: 60_000, + maxBytes: 256 * 1024, + env: gnupgEnvironment(work), + }); + try { + inspectRecipient(parseColons(listed.stdout), pins, NOW); + return 'ok'; + } catch (error) { + if (error instanceof CustodyError) return error.code; + throw error; + } + }; + for (const [name, expected] of Object.entries({ + good: 'ok', + other: 'ok', + 'good-aead': 'ok', + rsa3072: 'ok', + rsa2048: 'RECIPIENT_UNUSABLE', + 'expired-primary': 'RECIPIENT_EXPIRED', + 'expired-subkey': 'RECIPIENT_EXPIRED', + 'revoked-primary': 'RECIPIENT_REVOKED', + 'revoked-subkey': 'RECIPIENT_REVOKED', + 'sign-only': 'RECIPIENT_UNUSABLE', + 'two-encryption-subkeys': 'RECIPIENT_UNEXPECTED', + })) { + assert.match(PINS[name].primary, /^[0-9A-F]{40}$/u, name); + assert.match(PINS[name].subkey, /^[0-9A-F]{40}$/u, name); + assert.equal(await classify(fixture(name), PINS[name]), expected, name); + } + const both = join(work, 'two-keys.asc'); + writeFileSync( + both, + Buffer.concat([readFileSync(fixture('good')), readFileSync(fixture('other'))]), + ); + assert.equal(await classify(both, PINS.good), 'RECIPIENT_UNEXPECTED'); + assert.equal(await classify(fixture('good'), PINS.other), 'RECIPIENT_MISSING'); + // A rotation that changed the key without updating the subkey pin. + assert.equal( + await classify(fixture('good'), { primary: PINS.good.primary, subkey: PINS.other.subkey }), + 'RECIPIENT_MISSING', + ); +}); + +test('no tracked file holds a PGP private-key block', () => { + const marker = ['BEGIN PGP', 'PRIVATE KEY BLOCK'].join(' '); + const repository = fileURLToPath(new URL('../../', import.meta.url)); + const found = spawnSync('git', ['grep', '-l', '-F', marker], { + cwd: repository, + encoding: 'utf8', + }); + assert.equal(found.status, 1, `tracked private-key blocks:\n${found.stdout}`); +}); + +test('encryption publishes exactly one ciphertext for the pinned subkey and leaves no plaintext', async () => { + const { run, calls } = recording(); + const outcome = await attempt({ run }); + assert.equal(outcome.error, undefined); + const name = NAME(); + assert.deepEqual(readdirSync(outcome.destination), [name]); + const bytes = readFileSync(join(outcome.destination, name)); + assert.deepEqual(outcome.record, { + name, + bytes: bytes.length, + sha256: outcome.record.sha256, + recipient: PINS.good, + gnupg: outcome.record.gnupg, + }); + assert.match(outcome.record.sha256, /^[0-9a-f]{64}$/u); + assert.ok(Object.isFrozen(outcome.record) && Object.isFrozen(outcome.record.recipient)); + assert.deepEqual(calls, ['version', 'list', 'encrypt', 'recipients']); + assert.deepEqual(readdirSync(outcome.root), []); +}); + +test('an ephemeral key decrypts the ciphertext to the container, and tar extracts the staging', async () => { + const ephemeral = ephemeralKey(); + const outcome = await attempt({ recipient: { file: ephemeral.file, pins: ephemeral.pins } }); + assert.equal(outcome.error, undefined); + const out = fresh('round-trip'); + mkdirSync(out); + const plain = join(out, 'container.tar'); + ephemeral.decrypt(join(outcome.destination, NAME()), plain); + assert.deepEqual(readFileSync(plain), container); + const extracted = join(out, 'staging'); + mkdirSync(extracted); + assert.equal(spawnSync('tar', ['-xf', plain, '-C', extracted]).status, 0); + assert.deepEqual(readPrivateManifest(extracted), readPrivateManifest(staging)); + for (const name of readdirSync(join(staging, 'maps'))) { + assert.deepEqual( + readFileSync(join(extracted, 'maps', name)), + readFileSync(join(staging, 'maps', name)), + ); + } + // Secret key material is never a recipient. + refused( + await attempt({ recipient: { file: ephemeral.secret, pins: ephemeral.pins } }), + 'RECIPIENT_SECRET', + ); + ephemeral.dispose(); + assert.deepEqual(processesNaming(ephemeral.home), [], 'the ephemeral agent is still running'); + assert.equal(existsSync(ephemeral.home), false); +}); + +test('every committed refusal fixture is refused before encryption, and AEAD after it', async () => { + for (const [name, code] of [ + ['expired-primary', 'RECIPIENT_EXPIRED'], + ['expired-subkey', 'RECIPIENT_EXPIRED'], + ['revoked-primary', 'RECIPIENT_REVOKED'], + ['revoked-subkey', 'RECIPIENT_REVOKED'], + ['sign-only', 'RECIPIENT_UNUSABLE'], + ['rsa2048', 'RECIPIENT_UNUSABLE'], + ['two-encryption-subkeys', 'RECIPIENT_UNEXPECTED'], + ]) { + const { run, calls } = recording(); + refused(await attempt({ run, recipient: { file: fixture(name), pins: PINS[name] } }), code); + assert.deepEqual(calls, ['version', 'list'], name); + } + // GnuPG picks OCB for a key that advertises AEAD; that profile is refused. + refused( + await attempt({ recipient: { file: fixture('good-aead'), pins: PINS['good-aead'] } }), + 'PROFILE', + ); +}); + +test("GnuPG's own refusal is a second layer behind the pre-check", async () => { + // A forged listing passes the pre-check for an expired key; the real GnuPG still refuses it. + const pins = PINS['expired-subkey']; + const forged = [key('pub'), fpr(pins.primary), key('sub'), fpr(pins.subkey)] + .map((fields) => fields.join(':')) + .join('\n'); + const { run, calls } = recording({ list: { replace: () => ({ stdout: forged, stderr: '' }) } }); + refused( + await attempt({ run, recipient: { file: fixture('expired-subkey'), pins } }), + 'ENCRYPTION_FAILED', + ); + assert.deepEqual(calls, ['version', 'list', 'encrypt']); +}); + +test('the status must be exactly one pinned, completed SEIPDv1 AES-256 encryption', async () => { + const rewrite = (change) => ({ + encrypt: { after: (result) => ({ ...result, stdout: change(result.stdout) }) }, + }); + for (const [code, change] of [ + ['STATUS', (text) => `${text}gpg: encrypted\n`], + ['STATUS', (text) => text.replace(/\[GNUPG:\] END_ENCRYPTION\n/u, '')], + ['STATUS', (text) => text.replace(/(\[GNUPG:\] BEGIN_ENCRYPTION[^\n]*\n)/u, '$1$1')], + [ + 'STATUS', + (text) => + text.replace(/(\[GNUPG:\] BEGIN_ENCRYPTION[^\n]*\n)(\[GNUPG:\] END_ENCRYPTION\n)/u, '$2$1'), + ], + ['STATUS', (text) => `${text}[GNUPG:] INV_RECP 0 recipient\n`], + ['STATUS', (text) => `${text}[GNUPG:] FAILURE encrypt 53\n`], + ['STATUS', (text) => text.replace(/\[GNUPG:\] KEY_CONSIDERED[^\n]*\n/u, '')], + [ + 'STATUS', + (text) => + text.replace(/KEY_CONSIDERED [0-9A-F]{40} 0/u, `KEY_CONSIDERED ${'A'.repeat(40)} 0`), + ], + ['STATUS', (text) => text.replace(/KEY_CONSIDERED ([0-9A-F]{40}) 0/u, 'KEY_CONSIDERED $1 1')], + ['PROFILE', (text) => text.replace(/BEGIN_ENCRYPTION 2 9/u, 'BEGIN_ENCRYPTION 0 9 2')], + ['PROFILE', (text) => text.replace(/BEGIN_ENCRYPTION 2 9/u, 'BEGIN_ENCRYPTION 2 7')], + ['PROFILE', (text) => text.replace(/BEGIN_ENCRYPTION 2 9/u, 'BEGIN_ENCRYPTION 2 9 2')], + ]) { + refused(await attempt(recording(rewrite(change))), code); + } +}); + +test('the ciphertext must name exactly the pinned subkey and nothing else', async () => { + const rewrite = (change) => ({ + recipients: { after: (result) => ({ ...result, stdout: change(result.stdout) }) }, + }); + for (const change of [ + (text) => text.replace(/(\[GNUPG:\] ENC_TO[^\n]*\n)/u, '$1$1'), + (text) => text.replace(/ENC_TO [0-9A-F]{16}/u, `ENC_TO ${'1'.repeat(16)}`), + (text) => text.replace(/ENC_TO [0-9A-F]{16}/u, `ENC_TO ${'0'.repeat(16)}`), + (text) => text.replace(/\[GNUPG:\] ENC_TO[^\n]*\n/u, ''), + ]) { + refused(await attempt(recording(rewrite(change))), 'RECIPIENTS'); + } + refused( + await attempt( + recording({ + recipients: { + replace: () => { + throw new Error('listing failed'); + }, + }, + }), + ), + 'RECIPIENTS', + ); +}); + +test('the work directory must hold exactly the expected assets, the ciphertext within its bound', async () => { + const output = (args) => valueOf(args, '--output'); + for (const change of [ + (args) => writeFileSync(join(dirname(output(args)), 'extra'), 'planted'), + (args) => rmSync(output(args)), + (args) => appendFileSync(output(args), Buffer.alloc(container.length + 64 * 1024)), + ]) { + refused( + await attempt( + recording({ + encrypt: { + after: (result, args) => { + change(args); + return result; + }, + }, + }), + ), + 'ASSET', + ); + } +}); + +test('a GnuPG home that gains configuration, keyrings or secrets refuses', async () => { + for (const [kind, name] of [ + ['encrypt', 'common.conf'], + ['encrypt', 'pubring.kbx'], + ['list', 'gpg.conf'], + ['recipients', 'trustdb.gpg'], + ]) { + refused( + await attempt( + recording({ + [kind]: { + after: (result, args) => { + writeFileSync(join(valueOf(args, '--homedir'), name), 'planted'); + return result; + }, + }, + }), + ), + 'HOME_STATE', + ); + } +}); + +test('a failing GnuPG call or an unreadable recipient refuses with a fixed code', async () => { + const fail = () => { + throw new Error('child failed'); + }; + refused(await attempt(recording({ encrypt: { replace: fail } })), 'ENCRYPTION_FAILED'); + refused(await attempt(recording({ list: { replace: fail } })), 'RECIPIENT_UNREADABLE'); + refused(await attempt(recording({ version: { replace: fail } })), 'GPG_UNAVAILABLE'); + refused( + await attempt({ recipient: { file: join(fresh('none'), 'key.asc'), pins: PINS.good } }), + 'RECIPIENT_UNREADABLE', + ); + const empty = join(fresh('empty-recipient'), 'key.asc'); + mkdirSync(dirname(empty)); + writeFileSync(empty, ''); + const outcome = await attempt({ recipient: { file: empty, pins: PINS.good } }); + assert.ok( + ['RECIPIENT_UNREADABLE', 'RECIPIENT_MISSING'].includes(outcome.error?.code), + String(outcome.error), + ); + refused(outcome, outcome.error.code); +}); + +test('a ciphertext is published once: an existing one refuses before any GnuPG call', async () => { + const { root, destination } = places(); + writeFileSync(join(destination, NAME()), 'earlier ciphertext'); + const { run, calls } = recording(); + await assert.rejects( + encryptOnce({ + container, + recipient: { file: fixture('good'), pins: PINS.good }, + destination, + privateRoot: root, + run, + }), + refuses('ALREADY_ENCRYPTED'), + ); + assert.deepEqual(calls, []); + assert.equal(readFileSync(join(destination, NAME()), 'utf8'), 'earlier ciphertext'); + // One that appears while encrypting is never overwritten. + const raced = places(); + const late = recording({ + recipients: { + before: () => writeFileSync(join(raced.destination, NAME()), 'raced ciphertext'), + }, + }); + await assert.rejects( + encryptOnce({ + container, + recipient: { file: fixture('good'), pins: PINS.good }, + destination: raced.destination, + privateRoot: raced.root, + run: late.run, + }), + refuses('ALREADY_ENCRYPTED'), + ); + assert.equal(readFileSync(join(raced.destination, NAME()), 'utf8'), 'raced ciphertext'); + assert.deepEqual(readdirSync(raced.root), []); +}); + +test('the destination and the private root are separate directories, the root owner-only', async () => { + const { run, calls } = recording(); + const { root, destination } = places(); + const nested = join(root, 'destination'); + mkdirSync(nested); + const options = (overrides) => ({ + container, + recipient: { file: fixture('good'), pins: PINS.good }, + run, + ...overrides, + }); + await assert.rejects( + encryptOnce(options({ destination: nested, privateRoot: root })), + refuses('PLACEMENT'), + ); + const inner = join(destination, 'root'); + mkdirSync(inner); + chmodSync(inner, 0o700); + await assert.rejects( + encryptOnce(options({ destination, privateRoot: inner })), + refuses('PLACEMENT'), + ); + if (posixModes) { + const linked = fresh('linked-destination'); + symlinkSync(destination, linked); + await assert.rejects( + encryptOnce(options({ destination: linked, privateRoot: root })), + refuses('PLACEMENT'), + ); + const open = fresh('open-root'); + mkdirSync(open); + chmodSync(open, 0o755); + await assert.rejects( + encryptOnce(options({ destination, privateRoot: open })), + refuses('PRIVATE_ROOT'), + ); + } + await assert.rejects( + encryptOnce( + options({ container: Buffer.from('not a container'), destination, privateRoot: root }), + ), + (error) => error instanceof CustodyError && error.code.startsWith('CONTAINER_'), + ); + assert.deepEqual(calls, []); +}); + +test('no GnuPG output, path or host error text reaches a refusal', async () => { + const marker = join(fresh('marker'), 'private', 'path'); + const leaked = (error) => { + const text = JSON.stringify({ message: error.message, ...error, cause: error.cause }); + assert.equal(text.includes(marker), false, text); + assert.equal(text.includes('fixture.invalid'), false, text); + assert.match(error.message, /^PACKED_CLI_CUSTODY_[A-Z_]+$/u); + }; + const failing = recording({ + encrypt: { + replace: () => { + const error = new Error(`gpg failed at ${marker}`); + Object.assign(error, { + result: { stdout: marker, stderr: `good ${marker}` }, + }); + throw error; + }, + }, + }); + const first = await attempt(failing); + refused(first, 'ENCRYPTION_FAILED'); + leaked(first.error); + // A host failure after encryption is UNEXPECTED, never its message. + const lost = places(); + const vanished = recording({ + recipients: { before: () => rmSync(lost.destination, { recursive: true }) }, + }); + await assert.rejects( + encryptOnce({ + container, + recipient: { file: fixture('good'), pins: PINS.good }, + destination: lost.destination, + privateRoot: lost.root, + run: vanished.run, + }), + (error) => { + leaked(error); + return error instanceof CustodyError && error.code === 'UNEXPECTED'; + }, + ); + assert.deepEqual(vanished.calls, ['version', 'list', 'encrypt', 'recipients']); + assert.deepEqual(readdirSync(lost.root), []); +}); + +test( + 'an interrupt during encryption stops GnuPG, removes the plaintext and publishes nothing', + { skip: !posixModes }, + async () => { + const { root, destination } = places(); + const work = fresh('interrupt'); + mkdirSync(work); + const listing = join(work, 'listing.txt'); + const pins = PINS.good; + writeFileSync( + listing, + [key('pub'), fpr(pins.primary), key('sub'), fpr(pins.subkey)] + .map((fields) => fields.join(':')) + .join('\n'), + ); + const started = join(work, 'started'); + const pidFile = join(work, 'gpg.pid'); + const fakeGpg = join(work, 'gpg.mjs'); + writeFileSync( + fakeGpg, + [ + "import { readFileSync, writeFileSync } from 'node:fs';", + 'const args = process.argv.slice(2);', + "if (args.includes('--version')) process.stdout.write('gpg (GnuPG) 2.4.4\\n');", + `else if (args.includes('--show-keys')) process.stdout.write(readFileSync(${JSON.stringify(listing)}, 'utf8'));`, + `else { writeFileSync(${JSON.stringify(pidFile)}, String(process.pid)); writeFileSync(${JSON.stringify(started)}, ''); setInterval(() => {}, 1000); }`, + ].join('\n'), + ); + const driver = join(work, 'driver.mjs'); + const here = (name) => JSON.stringify(new URL(`./${name}`, import.meta.url).href); + writeFileSync( + driver, + [ + "import { readFileSync, rmSync } from 'node:fs';", + `import { encryptOnce } from ${here('custody.mjs')};`, + `import { exitOnInterrupt } from ${here('process.mjs')};`, + `const root = ${JSON.stringify(root)};`, + `await exitOnInterrupt(() => rmSync(root, { recursive: true, force: true }), () =>`, + ` encryptOnce({ container: readFileSync(${JSON.stringify(join(work, 'container.tar'))}),`, + ` recipient: { file: ${JSON.stringify(fixture('good'))}, pins: ${JSON.stringify(pins)} },`, + ` destination: ${JSON.stringify(destination)}, privateRoot: root,`, + ` gpg: { command: process.execPath, prefix: [${JSON.stringify(fakeGpg)}] } }));`, + ].join('\n'), + ); + writeFileSync(join(work, 'container.tar'), container); + const child = spawn(process.execPath, [driver], { stdio: 'ignore' }); + const closed = new Promise((resolve) => + child.once('close', (code, signal) => resolve({ code, signal })), + ); + for (let index = 0; index < 400 && !existsSync(started); index += 1) { + await new Promise((resolve) => setTimeout(resolve, 25)); + } + assert.ok(existsSync(started), 'the stand-in never started encrypting'); + const gpgPid = Number(readFileSync(pidFile, 'utf8')); + child.kill('SIGINT'); + assert.deepEqual(await closed, { code: 130, signal: null }); + assert.equal(existsSync(root), false, 'the private root and its plaintext survived'); + assert.deepEqual(readdirSync(destination), []); + assert.throws(() => process.kill(gpgPid, 0), { code: 'ESRCH' }); + }, +); From 0bd1953fa7c4031fbd41f02d122311569a7da6a8 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 08:59:29 +0300 Subject: [PATCH 69/97] docs(repo): record the Step 9b.1 implementation The packaging contract gains "Recipient custody implementation boundary - Step 9b.1": the strict USTAR dialect, the staging-relative private container, the isolated GnuPG runner (separate listing and encryption homes), the recipient pre-check order, encrypt-once with its status, profile, recipient, asset and no-clobber checks, plaintext and diagnostics rules, the synthetic fixtures and where the tests run. The tooling README summarizes the library. deferred-tasks.md re-records SEC-9A2A-2 to Step 9d, adds two Wave 4b key clauses (no AEAD feature; single-subkey export) and two named limits (no Windows run; the 2.2.8 floor enforced, never exercised). The phase register records the implementation checkpoint with its GnuPG 2.5.21 and 2.4.4 measurements and evidence, and NI's Windows run 1 (the probe saw 1 of 200 pairs collide as Numbers). Status lines name 9b.1 as under review. Refs: ADR-0111, Step 9b.1, NI Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 +- CLAUDE.md | 4 +- docs/reference/cli/packaging.md | 63 ++++++++++++++++++- docs/roadmap/current.md | 6 +- docs/roadmap/deferred-tasks.md | 27 +++++++- .../phase-2.5.5-hardening-and-remediation.md | 46 +++++++++++++- tools/cli-package/README.md | 9 ++- 7 files changed, 147 insertions(+), 12 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 92629c5e..f221495a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,7 +117,7 @@ explicit; source CI does not certify later documentation commits. The joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now -accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI is accepted too, and Steps 9b–19 have not started) and whole-wave acceptance remains Step 19. +accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI is accepted too, Step 9b.1 is implemented and under review, and Steps 9b.2–19 have not started) and whole-wave acceptance remains Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification @@ -130,7 +130,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25): round 1 reported no blocker, high or medium finding and no regression, with a clean security sitting; every product file and directory identity check now compares exact bigints, its low findings are recorded in deferred-tasks.md, and its Windows clause closes after three green advisory Windows runs ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)). Step 9b is next. `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25): round 1 reported no blocker, high or medium finding and no regression, with a clean security sitting; every product file and directory identity check now compares exact bigints, its low findings are recorded in deferred-tasks.md, and its Windows clause closes after three green advisory Windows runs ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)). Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b-preflight-decisions--2026-10-25)); Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-implementation-checkpoint--2026-10-25)). `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. See [docs/roadmap/current.md](docs/roadmap/current.md) for live status. diff --git a/CLAUDE.md b/CLAUDE.md index a248693f..653fa6ca 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -206,7 +206,7 @@ explicit; source CI does not certify later documentation commits. The joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now -accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI is accepted too, and Steps 9b–19 have not started) and whole-wave acceptance remains Step 19. +accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI is accepted too, Step 9b.1 is implemented and under review, and Steps 9b.2–19 have not started) and whole-wave acceptance remains Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification @@ -219,7 +219,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25): round 1 reported no blocker, high or medium finding and no regression, with a clean security sitting; every product file and directory identity check now compares exact bigints, its low findings are recorded in deferred-tasks.md, and its Windows clause closes after three green advisory Windows runs ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)). Step 9b is next. `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25): round 1 reported no blocker, high or medium finding and no regression, with a clean security sitting; every product file and directory identity check now compares exact bigints, its low findings are recorded in deferred-tasks.md, and its Windows clause closes after three green advisory Windows runs ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)). Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b-preflight-decisions--2026-10-25)); Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-implementation-checkpoint--2026-10-25)). `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. For live status, per-PR history, milestone dates, and open obligations, see the canonical diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 8b8c1c18..2941a217 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -1,6 +1,6 @@ # CLI Packaging Contract -- **Status**: Compiled graph, installed package and Step 8 native activation independently accepted on `development`; the Step 9a.1 candidate build and finalizer are independently accepted on `development` (2026-10-24); Step 9a.2a's strict decoder, private manifest and symbolication are independently accepted on `development` (2026-10-24); Step 9a.2b's product map controls, post-receipt rebuild refusals and `package_test` receipt are independently accepted on `development` (2026-10-24); the remaining Step 9 custody and release gates remain staged +- **Status**: Compiled graph, installed package and Step 8 native activation independently accepted on `development`; the Step 9a.1 candidate build and finalizer are independently accepted on `development` (2026-10-24); Step 9a.2a's strict decoder, private manifest and symbolication are independently accepted on `development` (2026-10-24); Step 9a.2b's product map controls, post-receipt rebuild refusals and `package_test` receipt are independently accepted on `development` (2026-10-24); Step 9b.1's private container, isolated GnuPG runner and encrypt-once library are implemented on `development` and under review (synthetic recipients only); the remaining Step 9 custody and release gates remain staged - **Date**: 2026-10-15 - **Related**: [ADR-0111](../../decisions/0111-cli-meta-operations-use-a-lazy-packaged-runtime.md), [commands.md](commands.md), [release runbook](../../runbooks/release-a-surface.md) @@ -513,6 +513,67 @@ update for future candidates; preserve historical fingerprint/decryption capabil for supported versions. Existing public ciphertext cannot be recalled by revocation. This wave creates no automatic deletion policy or production key. +### Recipient custody implementation boundary — Step 9b.1 + +Step 9b.1 implements the container and the one-time encryption as a library with synthetic +controls only. It holds no production recipient, encrypts no product staging, and changes +neither `producePackage`, `sealPackage`, the package-test receipt, the interim release guard +nor either workflow; encryption in the originating job is Step 9d's, reuse, the release +binding receipt and the production loader are Step 9b.2's. Its review and acceptance are +recorded in the [Wave 3 register](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments). + +- **One strict dialect.** `ustar.mjs` owns the USTAR dialect the archive inspector reads: + regular files only, every header field fixed, zero owner ids and device numbers, an exact + end and zero padding. The inspector delegates to it unchanged. +- **The private container.** `container.mjs` builds one owner-only (0600) USTAR from the + sealed staging in its own layout: `finalized.json`, `manifest.json`, `metafile-esm.json` and + `maps/.js.map`, sorted, with no host path. The staging must hold exactly those records + and the manifest's maps as regular files of at most 16 MiB each; every map must be the one the + manifest and the finalizer's record name, at most 259 entries and 128 MiB in total. Reading a + container back applies the same checks, and a decrypted container extracted with `tar -x` is a + staging the manifest reader consumes unchanged. +- **The isolated runner.** `gnupg.mjs` resolves `gpg` from absolute PATH entries only and + refuses one older than GnuPG 2.2.8. Every call runs in a fresh owner-only home the tool + creates, with `--no-options --no-autostart --no-keyring`, the C locale and an isolated + environment, bounded by `runCommand`. The recipient listing (`--show-keys`) uses its own home + and the default trust model, because GnuPG 2.4 cannot list an expired or revoked key under + `--trust-model always` without a trust database; that home may gain only GnuPG's own + `trustdb.gpg`. Encryption uses `--recipient-file`, `--trust-model always` (the pinned + fingerprints and the colon pre-check are the assurance), an AES-256 preference, no + compression and a fixed literal name, in a home that may gain only `random_seed`. +- **The recipient pre-check.** The recipient file is read bounded and copied, so the bytes + checked are the bytes encrypted to, and pins are exactly two 40-digit uppercase fingerprints + (OpenPGP v4). An armored secret key refuses from its bytes. The colon listing then refuses, + in this order: secret material, a missing pinned primary, more than one key, a missing pinned + subkey, revocation, expiry at the current time, unusability (invalid or disabled, no + encryption capability, future creation, or an encryption subkey other than cv25519 or RSA of + at least 3072 bits) and another usable encryption subkey. GnuPG's own refusal remains a + second, independent layer. +- **Encrypt once.** `encryptOnce` refuses before any GnuPG call when the destination already + holds `relavium--private-maps.tar.gpg`, when the destination and the owner-only + private root overlap, or when the root is not owner-only. Success requires exit 0, one + `KEY_CONSIDERED` for the pinned primary, one `BEGIN_ENCRYPTION 2 9` (SEIPDv1 with MDC and + AES-256; an AEAD profile refuses) before one `END_ENCRYPTION`, exactly the expected work + files, a ciphertext within the container's size plus 64 KiB whose public-key packets name + exactly the pinned subkey and no hidden recipient, and homes holding only what GnuPG creates. + The ciphertext is then published with `wx`, never over an existing file. It returns a frozen + custody record: name, size, SHA-256, pins and GnuPG version. +- **Plaintext and diagnostics.** The plaintext container and the recipient copy exist only in + an owner-only work directory under the caller's private root, removed on every path; an + interrupt in the caller's `exitOnInterrupt` scope stops GnuPG and removes the root. No secure + erasure is claimed. Every refusal is a fixed `PACKED_CLI_CUSTODY_`; GnuPG's stdout and + stderr, paths and host error text never surface. +- **Synthetic fixtures.** `custody-fixtures/` commits armored public v4 keys under + `fixture.invalid` for every class above, with their pins and a maintainer-only generator that + is never run in CI. No secret key is committed: the decryption round trip generates one per + run in a short owner-only home and stops its agent afterwards, and a test refuses any tracked + PGP private-key block. +- **Where it runs.** The custody tests are part of `test:acceptance-tools`, so they run in local + `pnpm run ci` and the four Ubuntu jobs inside `ci`'s needs, against the runner's GnuPG; they + fail, never skip, without GnuPG ≥ 2.2.8 (`docs/runbooks/local-dev-setup.md`). The Windows image + has no GnuPG, so they do not run there, and the 2.2.8 floor is enforced by the version check + without being exercised. + ## Package-test and release modes **Package-test mode** is required on PR/push and manual dry-run. In its isolated diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index 5b50c869..bacbcb92 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -138,7 +138,7 @@ The [deferred register](deferred-tasks.md) retains its existing limits; closure **Wave 3 Steps 1–8 merged to `main` on 2026-10-18 as [PR #95](https://github.com/HodeTech/Relavium/pull/95), merge commit `6d2a7372` (22:44 UTC). Wave 3 itself is not complete: Steps 9–19 had not started at -that merge (Steps 9a.1, 9a.2a and 9a.2b are now accepted on `development`, the native-identity increment NI is accepted too, and Steps 9b–19 have not started) and whole-wave acceptance +that merge (Steps 9a.1, 9a.2a and 9a.2b are now accepted on `development`, the native-identity increment NI is accepted too, Step 9b.1 is implemented and under review, and Steps 9b.2–19 have not started) and whole-wave acceptance remains Step 19.** The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) joins the completed scope, every review attribution, the independent review's dispositions and the publication state observed at merge. It was the maintainer-authorized, scoped publication checkpoint. @@ -172,7 +172,7 @@ bar and round cap, as its medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, by the author alone). Its other findings are recorded in -[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is independently accepted on `development` at `cf97b7f1` (2026-10-24): its round 13 reported no blocker, high or medium finding and its security sitting was clean, and the round's low findings are recorded in deferred-tasks.md ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)); Step 9a.2 is split into 9a.2a and 9a.2b, and Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24) after three rounds, the third with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)); Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24) after two rounds, the second with no blocker, high or medium finding and no regression ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)), and the native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25) after one round with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)), its Windows clause open until three green advisory Windows runs; the substrate's +[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is independently accepted on `development` at `cf97b7f1` (2026-10-24): its round 13 reported no blocker, high or medium finding and its security sitting was clean, and the round's low findings are recorded in deferred-tasks.md ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)); Step 9a.2 is split into 9a.2a and 9a.2b, and Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24) after three rounds, the third with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)); Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24) after two rounds, the second with no blocker, high or medium finding and no regression ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)), and the native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25) after one round with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)), its Windows clause open until three green advisory Windows runs; Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b-preflight-decisions--2026-10-25)), and Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is implemented on `development` and under review ([checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-implementation-checkpoint--2026-10-25)); the substrate's 9a.1–9d increments carry a dedicated archive/crypto security sitting. Steps 10–19 follow the [increment table](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) and end with whole-wave acceptance at Step 19. `development` is pushed after each accepted increment; PR #95 no longer bounds the @@ -1141,7 +1141,7 @@ an ESLint selector rather than a list. ### Wave 3 — Clear the ground -**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Steps 9a.1, 9a.2a, 9a.2b and NI accepted on `development` (2026-10-24/25), Steps 9b–19 NOT STARTED — 2026-10-25.** +**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Steps 9a.1, 9a.2a, 9a.2b and NI accepted on `development` (2026-10-24/25), Step 9b.1 implemented and under review, Steps 9b.2–19 NOT STARTED — 2026-10-25.** ADR-0110–0113, their linked contracts and the nineteen-increment plan were approved on 2026-10-16. [Steps 1–8 were independently accepted](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) on `development`, with whole Step 8 reviewed at `c83c8575`, and published as the scoped checkpoint diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index 577d58c4..dd24e251 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -689,7 +689,9 @@ that shows the shape. **Re-check:** when Step 9b designs the custody job, run the capture child and every descendant under an OS sandbox or a separate user that cannot read `staging`. *(low · tools/cli-package/symbolicate.mjs)* *Re-checked after rounds 1 and 2 (2026-10-24): the module confinement narrows the limit; the record and - packaging.md now say the refusals are in-process only.* + packaging.md now say the refusals are in-process only.* *Re-recorded 2026-10-25 (Step 9b preflight + decisions): Step 9b adds a library and synthetic controls and designs no custody job, so the trigger moves + to Step 9d, which designs the release job.* - [ ] **Two packs of one source can differ in the packed manifest's key order.** *Low, for Step 9c.* *(rigor pass, integration lens)* Two productions of the same tree packed the five `@relavium/*` `devDependencies`, which pnpm rewrites from `workspace:*` to versions at pack time, in different orders (`core, llm, shared, @@ -778,6 +780,23 @@ no code (INT-9A2B-R2-1, INT-9A2B-R2-2, CRIT-9A2B-R2-1, the last also amending th producer's owner-only root) until the next build. **Re-check:** with the composition test of the F5 entry, assert `dist` holds no `.map` file after each refused control build. *(low · tools/cli-package/map-controls-core.mjs)* +## Wave 3 Step 9b.1 — named limits (2026-10-25) + +Recorded with the Step 9b.1 implementation +([checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-implementation-checkpoint--2026-10-25)). +**Trigger for every entry:** the next change to the file it names, or a release producer on that platform. + +- [ ] **The custody tests do not run on Windows.** *Low; a named limit.* The Windows runner image has no + GnuPG, and the custody tests run only where `test:acceptance-tools` runs (local `pnpm run ci` and the + Ubuntu jobs). Local Windows developers need Gpg4win, as `docs/runbooks/local-dev-setup.md` says. + **Re-check:** before a release producer runs on Windows, install GnuPG there and run the custody tests + on that runner. *(low · tools/cli-package/gnupg.mjs)* +- [ ] **The GnuPG 2.2.8 floor is enforced, never exercised.** *Low; a named limit.* The version check + refuses an older GnuPG, but the tests ran only on 2.5.21 (local) and 2.4.4 (Ubuntu 24.04), so neither + encryption nor the `gpg -d | tar -x` recovery is proven on 2.2.8. **Re-check:** run the custody tests + against GnuPG 2.2.8 (for example in a container that ships it) before a recovery runbook names that + version. *(low · tools/cli-package/gnupg.mjs)* + ## Wave 3 NI — recorded low findings (2026-10-25) Round 1 of the native-identity increment at `fc5af701` accepted it with these lows @@ -2652,7 +2671,11 @@ future test cannot silently re-acquire it. the confirmed production recipient for encrypted release custody must be an OpenPGP v4 key with a cv25519 or RSA≥3072 encryption-capable subkey. Wave 3 Step 9 implements custody with synthetic keys only; an actual release fails closed until this recipient, its offline - recovery check and backup ownership exist. + recovery check and backup ownership exist. *Added 2026-10-25 (Step 9b preflight decisions):* the key + must not advertise the AEAD feature, because custody requires SEIPDv1 with MDC and AES-256 so that + `gpg -d | tar -x` with GnuPG ≥ 2.2.8 stays true; and the committed recipient file is the export of the + primary with only the pinned encryption subkey (`gpg --armor --export '!'`), + because a second usable encryption subkey refuses. - [ ] **Scheduled → Wave 5a: alt-screen word wrapping (#61).** The maintainer approved retaining character-wrap replacement with render-v2/D18 and picker diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index e40533a4..0b6828c1 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -3703,7 +3703,7 @@ No credential values or request captures belong in tracked evidence. | 9a.2b | **Recorded split, 2026-10-24 (maintainer-approved) — product map controls and the package-test receipt:** `check:map-controls` over product copies (wrong JS by line and column, wrong map, permuted sources, wrong source, missing, malformed and truncated upstream maps, a shifted valid upstream map, and the package-test producer's post-receipt rebuild refusals that ADR-0111 lists), run in `pnpm run ci` and as a required uncached CI job inside `ci`'s needs (the required contexts stay `ci` and `coverage`); then the atomic `package_test` flip across seal, binding, consumers, aggregate and both workflows, version 1 kept. Security sitting. *Preflight decisions approved on 2026-10-24 ([decisions](#wave-3-step-9a2b-preflight-decisions--2026-10-24)); independently accepted on `development` at `0f73c253` on 2026-10-24 ([checkpoint](#wave-3-step-9a2b-implementation-checkpoint--2026-10-24), [acceptance](#wave-3-step-9a2b-acceptance--2026-10-24)).* | | NI | **Native file identity on Windows (maintainer-approved 2026-10-24; after 9a.2b, before 9b):** every file and directory identity check compares `dev`/`ino` read with `{ bigint: true }` — the filesystem policy's pinned roots, the workflow catalog's open check and the media store's pins, read check and temporary claims — so a replaced file or directory can never compare equal on Windows, where `ino` is a 64-bit NTFS file ID that a JavaScript number cannot hold exactly; a replacement test per site. Found by 9a.2a's push CI ([acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)); recorded in [deferred-tasks.md](../deferred-tasks.md#native-file-identity-on-windows--approved-increment-2026-10-24). Security sitting. *Preflight decisions approved on 2026-10-25 ([decisions](#wave-3-ni-preflight-decisions--2026-10-25)); independently accepted on `development` at `fc5af701` on 2026-10-25 ([checkpoint](#wave-3-ni-implementation-checkpoint--2026-10-25), [acceptance](#wave-3-ni-acceptance--2026-10-25)); its Windows clause closes after three green advisory Windows runs on pushed heads.* | | 9b | **Recorded split — recipient custody:** synthetic-only isolated GnuPG controls (public-only home/keyring, pinned primary and encryption-subkey fingerprints, machine-readable refusal of missing, expired, revoked or unusable recipients), encrypt once and reuse the original ciphertext. Explicit crypto/custody security sitting. Fixture proof never claims production recipient or custody configuration. | -| 9b.1 | **Recorded split, 2026-10-25 (maintainer-approved) — container, isolated GnuPG runner and encrypt-once:** the strict USTAR dialect extracted from the archive reader with its writer promoted; the private container built from the sealed staging in a staging-relative layout, each record's digest and size checked against the private manifest; an isolated public-only GnuPG runner (GnuPG ≥ 2.2.8; `--recipient-file` with no keyring, agent or trustdb); a colon-listing pre-check that refuses missing, expired, revoked, unusable (v4 with cv25519 or RSA ≥ 3072 only) or unexpected recipients and secret material; `encryptOnce` (SEIPDv1 with MDC and AES-256, exactly one `ENC_TO` for the pinned subkey, the exact produced asset, a no-clobber publish, plaintext cleanup on every path); committed synthetic public fixtures, an ephemeral round-trip key, and the GnuPG prerequisite documented. Library and synthetic controls only: product paths and workflows stay unchanged. Crypto/custody security sitting. | +| 9b.1 | **Recorded split, 2026-10-25 (maintainer-approved) — container, isolated GnuPG runner and encrypt-once:** the strict USTAR dialect extracted from the archive reader with its writer promoted; the private container built from the sealed staging in a staging-relative layout, each record's digest and size checked against the private manifest; an isolated public-only GnuPG runner (GnuPG ≥ 2.2.8; `--recipient-file` with no keyring, agent or trustdb); a colon-listing pre-check that refuses missing, expired, revoked, unusable (v4 with cv25519 or RSA ≥ 3072 only) or unexpected recipients and secret material; `encryptOnce` (SEIPDv1 with MDC and AES-256, exactly one `ENC_TO` for the pinned subkey, the exact produced asset, a no-clobber publish, plaintext cleanup on every path); committed synthetic public fixtures, an ephemeral round-trip key, and the GnuPG prerequisite documented. Library and synthetic controls only: product paths and workflows stay unchanged. Crypto/custody security sitting. *Implemented on `development` and under review ([decisions](#wave-3-step-9b-preflight-decisions--2026-10-25), [checkpoint](#wave-3-step-9b1-implementation-checkpoint--2026-10-25)).* | | 9b.2 | **Recorded split, 2026-10-25 (maintainer-approved) — reuse, binding receipt and scope separation:** retained-ciphertext reuse without re-encryption (digest and size verified, zero GnuPG calls); the release binding receipt's schema v1, its validator and a sealer that writes only after the ciphertext exists (9d adds the origin, map-check and `ci`-run binding); the production recipient loader, absent in Wave 3; synthetic and production scope separation. Crypto/custody security sitting. | | 9c | **Recorded split — offline release state controller:** permission-enforcing offline GitHub/npm controller, permissioned prebuild discovery and reservation, and every rerun, partial, conflict and expiry state, with no overwrite, rebuild or re-encryption on uncertain history. | | 9d | **Recorded split — workflow, runbook and whole-Step-9 landing:** least-privilege actual workflow dependency graph, required artifact/platform aggregation and exact-artifact consumers, lost-staging refusal and the release runbook update. The whole-Step-9 cumulative review retains all 9a–9d changes with preservation traces into Step 8, and the dedicated archive/crypto security sitting closes the step. | @@ -10051,6 +10051,12 @@ The round's verified findings were four lows and four nits: The NI entry stays open until its Windows clause closes: three consecutive green advisory Windows runs on pushed heads, with the probe's logged counts recorded. Step 9b is next. +**Windows clause, run 1 of 3 (2026-10-25).** Push CI +[run 40151763770](https://github.com/HodeTech/Relavium/actions/runs/40151763770) at `c8c0d41b` passed all eighteen +jobs, the advisory Windows job among them. Its probe logged `1 of 200 replacement pairs collide as Numbers; 0 read +negative`: on the runner itself a replaced directory's file ID rounded to its original's once in 200 pairs, the +diagnosed mechanism, while every pair stayed distinct as exact identities, and the repeated replacement tests passed. + ### Wave 3 Step 9b preflight decisions — 2026-10-25 A read-only preflight (four readers and a synthesis at `c8c0d41b`) mapped Step 9b's obligations, the code it meets and @@ -10084,3 +10090,41 @@ Engineering defaults recorded with the rulings: - Windows has no GnuPG on its image, and the 2.2.8 floor is enforced only by the version check; both are recorded limits. - Before the first external round, the custody tests run in a local `ubuntu:24.04` container with the distribution's GnuPG, to gather 2.4.4 evidence. +### Wave 3 Step 9b.1 implementation checkpoint — 2026-10-25 + +**Scope and source.** Step 9b.1 (the 9b.1 row; #46, ADR-0111) is implemented on `development` after the +[Step 9b preflight decisions](#wave-3-step-9b-preflight-decisions--2026-10-25) (`12883df4`), in five ordered commits +and this record: + +1. **One strict USTAR dialect.** `ustar.mjs` extracts the archive inspector's reader with a path rule, a mode rule, + an entry hook and a refusal reporter, and promotes the writer. The inspector delegates with its refusal codes and + order unchanged, and `archive.test.mjs` and `seal.test.mjs` pass unchanged. +2. **The private container.** `container.mjs` builds and reads the owner-only container in the approved + staging-relative layout, every map checked against the manifest and the finalizer's record. +3. **The isolated GnuPG runner.** `gnupg.mjs`, with GnuPG ≥ 2.2.8 added to the local prerequisites. +4. **Synthetic fixtures.** Committed armored public keys for every recipient class, their pins and a + maintainer-only generator; the round-trip key is generated per run. +5. **Encrypt once.** `custody.mjs`: pins, the colon pre-check, `encryptOnce` and its tests. + +The [packaging contract](../../reference/cli/packaging.md#recipient-custody-implementation-boundary--step-9b1) records +the mechanism, the tooling README and deferred-tasks.md (SEC-9A2A-2 re-recorded to Step 9d, two Wave 4b key +clauses, two named limits) the rest. Product paths, the receipt, the guard and both workflows are unchanged; no +runtime dependency or ADR is added. + +**Measured before writing the parsers.** One script listed every fixture, encrypted to each and listed the +ciphertext's recipients on GnuPG 2.5.21 (local) and 2.4.4 (`ubuntu:24.04`): +- `BEGIN_ENCRYPTION 2 9` for a key without the AEAD feature, `0 9 2` (OCB) for one with it, on both versions. +- `--list-only` exits 0 with one `ENC_TO`; `--list-packets` exits 2. +- `INV_RECP` reasons do not tell expired, revoked and sign-only apart, and the `FAILURE` numbers differ between + calls, so neither is keyed on. +- An `INV_RECP` line carries the recipient file's absolute path, which is why status output is never surfaced. +- GnuPG 2.4.4 cannot list an expired or revoked key under `--trust-model always` without a trust database (`Fatal: + can't open trustdb.gpg`), so the listing runs in its own home with the default trust model. +- With `--trust-model always`, the encryption home gains only `random_seed`. + +**Author evidence at `85a9aebe`.** +- The four custody test files passed locally: 32 tests on GnuPG 2.5.21. +- They also passed in a clean clone in `ubuntu:24.04`: 32 of 32 with none skipped, as a non-root user, on its + GnuPG 2.4.4 after a fresh frozen install, with no GnuPG process left afterwards. +- The archive, seal, packed-CLI pin and tooling typecheck and lint suites passed. + diff --git a/tools/cli-package/README.md b/tools/cli-package/README.md index 0116717c..64f2f5e3 100644 --- a/tools/cli-package/README.md +++ b/tools/cli-package/README.md @@ -77,7 +77,14 @@ is written (`verifySealedDist`). `pnpm check:map-controls` (`map-controls.mjs` o `map-controls-core.mjs`) runs every named negative control on product copies of one real production, prints one JSON summary of codes and fixed lines (no absolute path, frame or map byte) and exits 1 unless every expected control ran once with its expected outcome; the required `map-controls` CI job -runs it uncached and uploads nothing. No private map, +runs it uncached and uploads nothing. Step 9b.1's custody library (synthetic recipients only; nothing +calls it yet) builds one owner-only USTAR of the sealed staging (`container.mjs`, over the one strict +dialect in `ustar.mjs`) and encrypts it once to one pinned recipient with GnuPG ≥ 2.2.8 in fresh +isolated homes (`custody.mjs` over `gnupg.mjs`): a colon pre-check refuses missing, expired, revoked, +unusable or unexpected recipients and secret material, the ciphertext must be SEIPDv1 with MDC and +AES-256 naming exactly the pinned subkey, an existing ciphertext is never overwritten, the plaintext +work directory is removed on every path and every refusal is a fixed `PACKED_CLI_CUSTODY_*` code. +`custody-fixtures/` holds the committed synthetic public keys; no secret key is committed. No private map, metafile, manifest or per-source inventory enters the public artifact. The release workflow's `release-guard.mjs` refuses every artifact until verified release custody exists: it sets its exit code to 1 before it evaluates its rule (`release-guard-core.mjs`), so a rule From 45bc56547157d3172237b7cd1255b71ea4ce4601 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 09:06:40 +0300 Subject: [PATCH 70/97] fix(repo): close the Step 9b.1 mutation sweep An author mutation sweep removed each custody guard in turn (44 mutants across the GnuPG flags, the classification, the status, asset, recipient, placement and publish checks, the container and the USTAR reader). Three survived. The PATH-resolution test's relative entry never pointed at the planted gpg, so it could not catch a resolver that searched relative entries; it now does. The recipients check's separate all-zero comparison and the container's symbolic-link test on the maps directory were already implied (a hidden recipient's zero key ID never equals the pinned subkey's; lstat never reports a link as a directory), so both are removed. Refs: ADR-0111, Step 9b.1 Co-Authored-By: Claude Opus 5.5 --- tools/cli-package/container.mjs | 6 ++---- tools/cli-package/custody.mjs | 4 ++-- tools/cli-package/gnupg.test.mjs | 11 +++++++---- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/tools/cli-package/container.mjs b/tools/cli-package/container.mjs index 434f483f..c28f0e88 100644 --- a/tools/cli-package/container.mjs +++ b/tools/cli-package/container.mjs @@ -154,10 +154,8 @@ export function buildPrivateContainer(staging, limits = CONTAINER_LIMITS) { if (!isDeepStrictEqual(readdirSync(staging).sort(), [...RECORDS, 'maps'].sort())) { refuseCustody('CONTAINER_SET'); } - const mapsDirectory = lstatSync(join(staging, 'maps')); - if (!mapsDirectory.isDirectory() || mapsDirectory.isSymbolicLink()) { - refuseCustody('CONTAINER_SET'); - } + // lstat: a linked maps directory is not a directory. + if (!lstatSync(join(staging, 'maps')).isDirectory()) refuseCustody('CONTAINER_SET'); const names = readdirSync(join(staging, 'maps')).sort(); if (names.length > limits.entries - RECORDS.length) refuseCustody('CONTAINER_SIZE'); /** @type {Map} */ diff --git a/tools/cli-package/custody.mjs b/tools/cli-package/custody.mjs index 226bf716..0300440f 100644 --- a/tools/cli-package/custody.mjs +++ b/tools/cli-package/custody.mjs @@ -284,8 +284,8 @@ export async function encryptOnce({ ); if ( recipients.length !== 1 || - recipients[0].args[0] !== pins.subkey.slice(-16) || - recipients[0].args[0] === '0'.repeat(16) + // A hidden recipient's all-zero key ID never equals the pinned subkey's. + recipients[0].args[0] !== pins.subkey.slice(-16) ) { refuseCustody('RECIPIENTS'); } diff --git a/tools/cli-package/gnupg.test.mjs b/tools/cli-package/gnupg.test.mjs index f15b2913..6803f180 100644 --- a/tools/cli-package/gnupg.test.mjs +++ b/tools/cli-package/gnupg.test.mjs @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; -import { delimiter, join } from 'node:path'; +import { delimiter, isAbsolute, join, relative } from 'node:path'; import { after, test } from 'node:test'; import { CustodyError } from './custody-error.mjs'; import { @@ -35,13 +35,16 @@ test('gpg resolves from absolute PATH entries only and is required, never skippe }); assert.match(version, /^\d+\.\d+\.\d+$/u); assert.throws(() => resolveGpg(join(scratch, 'none')), refuses('GPG_UNAVAILABLE')); - // A relative entry is never searched, even when it holds a gpg. + // A relative entry is never searched, even when it resolves to a directory holding a gpg. mkdirSync(join(scratch, 'relative')); - writeFileSync(join(scratch, 'relative', 'gpg'), ''); + writeFileSync(join(scratch, 'relative', process.platform === 'win32' ? 'gpg.exe' : 'gpg'), ''); + const entry = relative(process.cwd(), join(scratch, 'relative')); + assert.equal(isAbsolute(entry), false); assert.throws( - () => resolveGpg(['relative', join(scratch, 'none')].join(delimiter)), + () => resolveGpg([entry, join(scratch, 'none')].join(delimiter)), refuses('GPG_UNAVAILABLE'), ); + assert.deepEqual(resolveGpg(join(scratch, 'relative')).prefix, []); }); test('the version floor is GnuPG 2.2.8, read from the version line only', async () => { From f0555a6176fc4c28a585e2911af68b880866cd43 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 10:15:52 +0300 Subject: [PATCH 71/97] fix(repo): close the Step 9b.1 internal rigor pass An internal pass at 45bc5654 (claims, platform, crypto/custody pre-sitting, tests and integration lenses with a completeness critic) confirmed seven medium findings, all corrected: - host errors before encryptOnce's main try block escaped with their path; the whole function now runs inside the fixed-code boundary (a missing destination refuses PLACEMENT, a missing root PRIVATE_ROOT, an unremovable work directory CLEANUP), with cleanup outside finally; - the AEAD control assumed GnuPG 2.3 or later; it now takes its expectation from the installed version and never skips; - the interrupt test raced the reaping of the killed stand-in (about 1 in 30 runs); it polls with a deadline (0 of 30); - the success test checked the record's digest and version against itself; - the MDC half of the profile check, the primary-as-subkey guard, the finalizer record's map binding, the checked recipient copy and its bound had no isolating test; each has one. Lows corrected: an atomic publish (a temporary sibling hard-linked to the final name), the isolation flags on the version call, the metafile parsed as a JSON object, a distinct pre-write total code, more USTAR, status and classification cases, and the contract's header-field, listed-key, future-creation, fixture-class and Windows wording (the Windows image ships Git's MSYS2 gpg; Windows developers use WSL). Two lows are recorded in deferred-tasks.md. A rerun sweep of 52 mutants left no survivor. Refs: ADR-0111, Step 9b.1 Co-Authored-By: Claude Opus 5.5 --- docs/reference/cli/packaging.md | 52 ++-- docs/roadmap/deferred-tasks.md | 26 +- .../phase-2.5.5-hardening-and-remediation.md | 44 +++ docs/runbooks/local-dev-setup.md | 2 +- tools/cli-package/container.mjs | 7 +- tools/cli-package/container.test.mjs | 90 +++++- tools/cli-package/custody.mjs | 291 +++++++++++------- tools/cli-package/custody.test.mjs | 146 ++++++++- tools/cli-package/gnupg.mjs | 2 +- tools/cli-package/gnupg.test.mjs | 21 ++ tools/cli-package/ustar.test.mjs | 5 + 11 files changed, 527 insertions(+), 159 deletions(-) diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 2941a217..342c9195 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -523,15 +523,18 @@ binding receipt and the production loader are Step 9b.2's. Its review and accept recorded in the [Wave 3 register](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments). - **One strict dialect.** `ustar.mjs` owns the USTAR dialect the archive inspector reads: - regular files only, every header field fixed, zero owner ids and device numbers, an exact - end and zero padding. The inspector delegates to it unchanged. + regular files only, every header field fixed except the modification time, whose syntax + alone is checked, zero owner ids and device numbers, an exact end and zero padding. The + inspector delegates to it unchanged. - **The private container.** `container.mjs` builds one owner-only (0600) USTAR from the sealed staging in its own layout: `finalized.json`, `manifest.json`, `metafile-esm.json` and `maps/.js.map`, sorted, with no host path. The staging must hold exactly those records - and the manifest's maps as regular files of at most 16 MiB each; every map must be the one the - manifest and the finalizer's record name, at most 259 entries and 128 MiB in total. Reading a - container back applies the same checks, and a decrypted container extracted with `tar -x` is a - staging the manifest reader consumes unchanged. + and the manifest's maps as regular files of at most 16 MiB each, at most 259 entries and + 128 MiB in total (the total refused before the archive is assembled). Every map's digest must + be the manifest's, and the finalizer's record must name exactly the manifest's outputs and + maps with each map's size. The manifest records no digest for the metafile, which must parse + as a JSON object. Reading a container back applies the same checks, and a decrypted container + extracted with `tar -x` is a staging the manifest reader consumes unchanged. - **The isolated runner.** `gnupg.mjs` resolves `gpg` from absolute PATH entries only and refuses one older than GnuPG 2.2.8. Every call runs in a fresh owner-only home the tool creates, with `--no-options --no-autostart --no-keyring`, the C locale and an isolated @@ -544,11 +547,13 @@ recorded in the [Wave 3 register](../../roadmap/phases/phase-2.5.5-hardening-and - **The recipient pre-check.** The recipient file is read bounded and copied, so the bytes checked are the bytes encrypted to, and pins are exactly two 40-digit uppercase fingerprints (OpenPGP v4). An armored secret key refuses from its bytes. The colon listing then refuses, - in this order: secret material, a missing pinned primary, more than one key, a missing pinned - subkey, revocation, expiry at the current time, unusability (invalid or disabled, no - encryption capability, future creation, or an encryption subkey other than cv25519 or RSA of - at least 3072 bits) and another usable encryption subkey. GnuPG's own refusal remains a - second, independent layer. + in this order: secret material, a missing pinned primary, more than one listed key, a missing + pinned subkey, revocation, expiry at the current time, unusability (invalid or disabled, no + encryption capability, a creation time in the future, or an encryption subkey other than + cv25519 or RSA of at least 3072 bits) and another usable encryption subkey. GnuPG's listing + omits a key it cannot use, such as one created in the future or one without a valid user ID, + so such a key refuses as missing or goes uncounted; GnuPG's own refusal and the + `KEY_CONSIDERED` and recipient checks remain the independent second layer for it. - **Encrypt once.** `encryptOnce` refuses before any GnuPG call when the destination already holds `relavium--private-maps.tar.gpg`, when the destination and the owner-only private root overlap, or when the root is not owner-only. Success requires exit 0, one @@ -556,23 +561,30 @@ recorded in the [Wave 3 register](../../roadmap/phases/phase-2.5.5-hardening-and AES-256; an AEAD profile refuses) before one `END_ENCRYPTION`, exactly the expected work files, a ciphertext within the container's size plus 64 KiB whose public-key packets name exactly the pinned subkey and no hidden recipient, and homes holding only what GnuPG creates. - The ciphertext is then published with `wx`, never over an existing file. It returns a frozen - custody record: name, size, SHA-256, pins and GnuPG version. + The ciphertext is then published atomically: written to a fresh sibling, then hard-linked to + its final name, which never replaces an existing file, so a failed write never leaves a + partial file under that name. It returns a frozen custody record: name, size, SHA-256, pins + and GnuPG version. - **Plaintext and diagnostics.** The plaintext container and the recipient copy exist only in an owner-only work directory under the caller's private root, removed on every path; an interrupt in the caller's `exitOnInterrupt` scope stops GnuPG and removes the root. No secure - erasure is claimed. Every refusal is a fixed `PACKED_CLI_CUSTODY_`; GnuPG's stdout and - stderr, paths and host error text never surface. + erasure is claimed. Every refusal is a fixed `PACKED_CLI_CUSTODY_`, including a host + failure (`UNEXPECTED`) and a work directory that cannot be removed (`CLEANUP`); GnuPG's stdout + and stderr, paths and host error text never surface. - **Synthetic fixtures.** `custody-fixtures/` commits armored public v4 keys under - `fixture.invalid` for every class above, with their pins and a maintainer-only generator that - is never run in CI. No secret key is committed: the decryption round trip generates one per + `fixture.invalid` for the classes GnuPG can produce — good cv25519 and RSA-3072 keys, one + advertising AEAD, an RSA-2048 subkey, an expired primary and subkey, a revoked primary and + subkey, a sign-only key and a key with two encryption subkeys — with their pins and a + maintainer-only generator that is never run in CI; the other classes are covered by crafted + listings. No secret key is committed: the decryption round trip generates one per run in a short owner-only home and stops its agent afterwards, and a test refuses any tracked PGP private-key block. - **Where it runs.** The custody tests are part of `test:acceptance-tools`, so they run in local `pnpm run ci` and the four Ubuntu jobs inside `ci`'s needs, against the runner's GnuPG; they - fail, never skip, without GnuPG ≥ 2.2.8 (`docs/runbooks/local-dev-setup.md`). The Windows image - has no GnuPG, so they do not run there, and the 2.2.8 floor is enforced by the version check - without being exercised. + fail, never skip, without GnuPG ≥ 2.2.8 (`docs/runbooks/local-dev-setup.md`). They do not run + on Windows: no Windows job runs `test:acceptance-tools`, the image's only `gpg` is Git for + Windows' MSYS2 build, never exercised here, and the tests use POSIX process and permission + tools. The 2.2.8 floor is enforced by the version check without being exercised. ## Package-test and release modes diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index dd24e251..073f148c 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -2,7 +2,7 @@ > Status: Living -> Last updated: 2026-10-25 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Step 9a.1 is independently accepted on `development` with its recorded low findings below, Step 9a.2a is independently accepted on `development` with its recorded low findings below, Step 9a.2b is independently accepted on `development` with its recorded low findings below, the native-identity increment NI independently accepted on `development` with its recorded low findings below (its Windows clause open), and Steps 9b–19 not started; the PR95 increment is closed on `development` under the maintainer's 2026-10-21 round cap). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's edge cases with the maintainer's 2026-10-21 acceptance bar and round cap and the 2026-10-22 rulings. Existing approved deferrals remain. +> Last updated: 2026-10-25 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Step 9a.1 is independently accepted on `development` with its recorded low findings below, Step 9a.2a is independently accepted on `development` with its recorded low findings below, Step 9a.2b is independently accepted on `development` with its recorded low findings below, the native-identity increment NI independently accepted on `development` with its recorded low findings below (its Windows clause open), Step 9b.1 implemented and under review with its named limits below, and Steps 9b.2–19 not started; the PR95 increment is closed on `development` under the maintainer's 2026-10-21 round cap). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's edge cases with the maintainer's 2026-10-21 acceptance bar and round cap and the 2026-10-22 rulings. Existing approved deferrals remain. > The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) records completed prerequisites; full M/N lifecycle and other scheduled work remain with their owners. > > Earlier triage: the Phase-2.6 rewrite triaged every open item, the 2026-07-19 full-project @@ -786,11 +786,25 @@ Recorded with the Step 9b.1 implementation ([checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-implementation-checkpoint--2026-10-25)). **Trigger for every entry:** the next change to the file it names, or a release producer on that platform. -- [ ] **The custody tests do not run on Windows.** *Low; a named limit.* The Windows runner image has no - GnuPG, and the custody tests run only where `test:acceptance-tools` runs (local `pnpm run ci` and the - Ubuntu jobs). Local Windows developers need Gpg4win, as `docs/runbooks/local-dev-setup.md` says. - **Re-check:** before a release producer runs on Windows, install GnuPG there and run the custody tests - on that runner. *(low · tools/cli-package/gnupg.mjs)* +- [ ] **The custody tests do not run on Windows.** *Low; a named limit.* No Windows job runs + `test:acceptance-tools`; the Windows image's only `gpg` is Git for Windows' MSYS2 build in `usr\bin`, + never exercised here, and the tests use POSIX process and permission tools (`ps`, owner-only modes), so + `docs/runbooks/local-dev-setup.md` sends Windows developers to WSL. **Re-check:** before a release + producer runs on Windows, run the custody tests on that runner and record which `gpg.exe` `resolveGpg` + selects and whether its Windows path handling passes. *(low · tools/cli-package/gnupg.mjs)* +- [ ] **An interrupted custody test run can leave its ephemeral key home.** *Low; test-only.* *(rigor pass, + SEC-9B1-PS-2)* `gpg-fixture.mjs` removes the round-trip key's home, its synthetic secret export and its + agent in `after()`, which a SIGINT to `node --test` skips: the `rg-*` and `rge-*` directories under the + temporary directory and an orphaned `gpg-agent` remain. The key is synthetic and never decrypts a + product archive. **Re-check:** stop the agent and remove both directories from a process exit hook as + well. *(low · tools/cli-package/gpg-fixture.mjs)* +- [ ] **The pre-check counts only the keys GnuPG lists.** *Low; fail-closed.* *(rigor pass, SEC-9B1-PS-5)* + `--show-keys` omits a key an import would reject, for example one whose user ID packet was removed, so a + recipient file holding such a key before the pinned one passes the one-key rule. GnuPG then encrypts to + the first key in the file and refuses it, and `KEY_CONSIDERED` and the recipient check refuse any other + outcome, so nothing is published. **Re-check:** count primary-key packets independently of import + filtering if a recipient file is ever produced by anything but `gpg --export`. *(low · + tools/cli-package/custody.mjs)* - [ ] **The GnuPG 2.2.8 floor is enforced, never exercised.** *Low; a named limit.* The version check refuses an older GnuPG, but the tests ran only on 2.5.21 (local) and 2.4.4 (Ubuntu 24.04), so neither encryption nor the `gpg -d | tar -x` recovery is proven on 2.2.8. **Re-check:** run the custody tests diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 0b6828c1..a58e409e 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -10128,3 +10128,47 @@ ciphertext's recipients on GnuPG 2.5.21 (local) and 2.4.4 (`ubuntu:24.04`): GnuPG 2.4.4 after a fresh frozen install, with no GnuPG process left afterwards. - The archive, seal, packed-CLI pin and tooling typecheck and lint suites passed. +**Internal rigor pass, 2026-10-25.** Two checks preceded the first round: +- an author mutation sweep at `0bd1953f`, 44 mutants; +- an internal pass at `45bc5654`, with claims, platform, crypto/custody pre-sitting, tests and integration lenses + and a completeness critic, every finding adversarially verified (the record is local). + +Commit `45bc5654` fixed the sweep's survivors. Its message says three survived; a fourth, the container's pre-write +total check, had been counted as killed only because a flaky interrupt test failed under it. + +The pass confirmed seven medium findings, all corrected: +- **Host errors escaped with their path.** Errors raised before `encryptOnce`'s main try block carried their + path: a missing destination or root, a failed listing or path resolution. The whole function now sits inside + the fixed-code boundary. A missing destination refuses `PLACEMENT`, a missing root `PRIVATE_ROOT`, and a work + directory that cannot be removed `CLEANUP`. +- **The AEAD control was version-blind.** It asserted `PROFILE`, which holds only on GnuPG 2.3 and later. It now + takes the expectation from the installed version, never skipping: GnuPG 2.2 has no AEAD, encrypts SEIPDv1 and + succeeds. +- **The interrupt test raced the reaping.** It probed the stand-in's process once, racing the reaping of its + killed group, and failed about once in 30 runs. It now polls with a deadline and failed 0 of 30. +- **The success test checked the record against itself.** The record's SHA-256 and GnuPG version are now + computed independently. +- **Three guards had no isolating test.** The MDC half of the profile check, the pinned-primary-as-subkey guard + and the finalizer record's map binding (path, digest and size) each have one now. +- **The checked recipient copy and its bound were unproven.** Nothing proved that the copy is the one encrypted + to, or that the 256 KiB bound holds. One test now changes the caller's file mid-run, and another exceeds the + bound. + +Low findings corrected: +- The publish is atomic: a temporary sibling, hard-linked to the final name. +- The version call carries the isolation flags. +- The metafile must parse as a JSON object, and the contract says what binds each record. +- The pre-write total refuses with its own code. +- More USTAR header, status-keyword and classification cases are tested. +- The contract's wording on the modification-time field, listed keys, future creation, the fixture classes and + Windows is corrected (the Windows image ships Git's MSYS2 `gpg`, and the tests use POSIX tools, so Windows + developers use WSL), and so is deferred-tasks.md's freshness line. + +Two lows are recorded in deferred-tasks.md: an interrupted test run can leave the ephemeral key's home, and the +pre-check counts only the keys GnuPG lists. A rerun sweep of 52 mutants left no survivor. + +**CI at `45bc5654`.** The full `pnpm run ci` failed one test outside the increment. The usage-coverage +inventory's `vitest list` collection exceeded its 30-second spawn timeout while the rigor pass's agents held the +load average at 40–66. On a quiet machine it passed three of three runs in 10–12 seconds. The full CI reruns at +the corrected head, alone, before the first round. + diff --git a/docs/runbooks/local-dev-setup.md b/docs/runbooks/local-dev-setup.md index de824a50..464839db 100644 --- a/docs/runbooks/local-dev-setup.md +++ b/docs/runbooks/local-dev-setup.md @@ -21,7 +21,7 @@ For the full package layout this runbook references, see | Rust toolchain (`rustup`, `cargo`) | Builds the Tauri v2 desktop backend | Only needed to build `apps/desktop`. The engine, CLI, and tests do not need Rust. | | Tauri OS prerequisites | Native WebView + build deps | macOS: Xcode Command Line Tools. Windows: WebView2 + MSVC build tools. Linux: WebKitGTK + libsecret dev packages. See the [Tauri prerequisites guide](https://v2.tauri.app/start/prerequisites/). | | Git | Workflows and agents are git-native YAML | — | -| GnuPG ≥ 2.2.8 (`gpg`) | The release custody tests (`pnpm run ci`) encrypt synthetic map containers | Required, never skipped: the tests fail without it. macOS: `brew install gnupg`. Debian/Ubuntu: `apt install gnupg` (the CI Ubuntu runner has it). Windows: Gpg4win; the CI runs these tests on Ubuntu only. Release tooling, never a runtime dependency. | +| GnuPG ≥ 2.2.8 (`gpg`) | The release custody tests (`pnpm run ci`) encrypt synthetic map containers | Required, never skipped: the tests fail without it. macOS: `brew install gnupg`. Debian/Ubuntu: `apt install gnupg` (the CI Ubuntu runner has it). Windows: run these tests under WSL, since they use POSIX process and permission tools; the CI runs them on Ubuntu only. Release tooling, never a runtime dependency. | > You can do meaningful work on the engine and CLI **without** the Rust/Tauri > toolchain. Install Rust only when you start on `apps/desktop`. diff --git a/tools/cli-package/container.mjs b/tools/cli-package/container.mjs index c28f0e88..a8e864b5 100644 --- a/tools/cli-package/container.mjs +++ b/tools/cli-package/container.mjs @@ -91,6 +91,10 @@ function checkContents(files) { refuseCustody('CONTAINER_DRIFT'); } } + // The manifest records no digest for the metafile; it must at least be the finalizer's JSON object. + if (!isRecord(json(/** @type {Buffer} */ (files.get('metafile-esm.json')), 'CONTAINER_RECORD'))) { + refuseCustody('CONTAINER_RECORD'); + } const record = json(/** @type {Buffer} */ (files.get('finalized.json')), 'CONTAINER_RECORD'); if ( !isRecord(record) || @@ -166,7 +170,8 @@ export function buildPrivateContainer(staging, limits = CONTAINER_LIMITS) { files.set(`maps/${name}`, read(join(staging, 'maps', name))); } const manifest = checkContents(files); - if (containerBytes(files) > limits.totalBytes) refuseCustody('CONTAINER_SIZE'); + // Refused before the archive is assembled in memory; the read-back check would refuse it after. + if (containerBytes(files) > limits.totalBytes) refuseCustody('CONTAINER_TOTAL'); const sorted = [...files].sort(([left], [right]) => (left < right ? -1 : 1)); const bytes = writeUstar(sorted, () => ENTRY_MODE); // The written container must read back as exactly these entries. diff --git a/tools/cli-package/container.test.mjs b/tools/cli-package/container.test.mjs index a0645ffd..114ac2b3 100644 --- a/tools/cli-package/container.test.mjs +++ b/tools/cli-package/container.test.mjs @@ -96,6 +96,89 @@ test('a staging that is not exactly the sealed records and maps refuses', () => }, ], ['CONTAINER_RECORD', (copy) => writeFileSync(join(copy, 'manifest.json'), '{"version":1}')], + [ + 'CONTAINER_RECORD', + (copy) => { + const path = join(copy, 'finalized.json'); + const record = JSON.parse(readFileSync(path, 'utf8')); + record.maps[0].bytes += 1; + writeFileSync(path, JSON.stringify(record)); + }, + ], + [ + 'CONTAINER_RECORD', + (copy) => { + const path = join(copy, 'finalized.json'); + const record = JSON.parse(readFileSync(path, 'utf8')); + record.maps[0].sha256 = '0'.repeat(64); + writeFileSync(path, JSON.stringify(record)); + }, + ], + [ + 'CONTAINER_RECORD', + (copy) => { + const path = join(copy, 'finalized.json'); + const record = JSON.parse(readFileSync(path, 'utf8')); + record.maps[0].path = 'dist/other.js.map'; + writeFileSync(path, JSON.stringify(record)); + }, + ], + [ + 'CONTAINER_RECORD', + (copy) => { + const path = join(copy, 'finalized.json'); + const record = JSON.parse(readFileSync(path, 'utf8')); + record.version = 2; + writeFileSync(path, JSON.stringify(record)); + }, + ], + [ + 'CONTAINER_RECORD', + (copy) => { + const path = join(copy, 'finalized.json'); + const record = JSON.parse(readFileSync(path, 'utf8')); + record.extra = true; + writeFileSync(path, JSON.stringify(record)); + }, + ], + [ + 'CONTAINER_RECORD', + (copy) => { + const path = join(copy, 'finalized.json'); + const record = JSON.parse(readFileSync(path, 'utf8')); + record.upstreamMaps = 1.5; + writeFileSync(path, JSON.stringify(record)); + }, + ], + [ + 'CONTAINER_RECORD', + (copy) => { + const path = join(copy, 'finalized.json'); + const record = JSON.parse(readFileSync(path, 'utf8')); + delete record.outputs[0].bytes; + writeFileSync(path, JSON.stringify(record)); + }, + ], + [ + 'CONTAINER_RECORD', + (copy) => { + const path = join(copy, 'finalized.json'); + const record = JSON.parse(readFileSync(path, 'utf8')); + record.outputs[0].bytes = '1'; + writeFileSync(path, JSON.stringify(record)); + }, + ], + [ + 'CONTAINER_RECORD', + (copy) => { + const path = join(copy, 'finalized.json'); + const record = JSON.parse(readFileSync(path, 'utf8')); + record.maps = {}; + writeFileSync(path, JSON.stringify(record)); + }, + ], + ['CONTAINER_RECORD', (copy) => writeFileSync(join(copy, 'metafile-esm.json'), 'not json')], + ['CONTAINER_RECORD', (copy) => writeFileSync(join(copy, 'metafile-esm.json'), '[]')], ]) { assert.throws(() => buildPrivateContainer(staging(change)), refuses(code), code); } @@ -145,9 +228,14 @@ test('the entry, count and total bounds refuse before a container is written', ( refuses('CONTAINER_SIZE'), ); const { bytes } = buildPrivateContainer(copy); + // The total is refused before the archive is assembled, not by the read-back check. assert.throws( () => buildPrivateContainer(copy, { ...CONTAINER_LIMITS, totalBytes: bytes.length - 1 }), - refuses('CONTAINER_SIZE'), + refuses('CONTAINER_TOTAL'), + ); + assert.throws( + () => readPrivateContainer(bytes, { ...CONTAINER_LIMITS, entries: 3 }), + refuses('CONTAINER_FILE_COUNT'), ); assert.throws( () => readPrivateContainer(bytes, { ...CONTAINER_LIMITS, totalBytes: bytes.length - 1 }), diff --git a/tools/cli-package/custody.mjs b/tools/cli-package/custody.mjs index 0300440f..4fa8b77b 100644 --- a/tools/cli-package/custody.mjs +++ b/tools/cli-package/custody.mjs @@ -9,7 +9,8 @@ * every path; the caller's interrupt scope stops GnuPG and removes that root. This library holds no * production recipient: Step 9b uses synthetic recipients only. */ -import { lstatSync, mkdtempSync, readdirSync, realpathSync, rmSync } from 'node:fs'; +import { randomUUID } from 'node:crypto'; +import { linkSync, lstatSync, mkdtempSync, readdirSync, realpathSync, rmSync } from 'node:fs'; import { isAbsolute, join, relative, sep } from 'node:path'; import { isDeepStrictEqual } from 'node:util'; import { readPrivateContainer } from './container.mjs'; @@ -151,9 +152,23 @@ function within(root, path) { return rel === '' || (!isAbsolute(rel) && rel !== '..' && !rel.startsWith(`..${sep}`)); } -/** An owner-only directory of this user, not a link. */ +/** A directory, not a link; a missing path is not one. */ +function plainDirectory(path) { + try { + return lstatSync(path).isDirectory(); + } catch { + return false; + } +} + +/** An owner-only directory of this user, not a link; a missing path is not one. */ function privateDirectory(path) { - const info = lstatSync(path); + let info; + try { + info = lstatSync(path); + } catch { + return false; + } return ( info.isDirectory() && !info.isSymbolicLink() && @@ -163,6 +178,29 @@ function privateDirectory(path) { ); } +/** + * Publish once and atomically: the bytes go to a fresh sibling, which is then hard-linked to the + * final name. A link never replaces an existing file, and a failed write never leaves a partial + * file under the final name. + * @param {string} directory @param {string} name @param {Buffer} bytes + */ +function publish(directory, name, bytes) { + const temporary = join(directory, `.${name}.${randomUUID()}.partial`); + try { + writePrivateFile(temporary, bytes); + try { + linkSync(temporary, join(directory, name)); + } catch (error) { + if (error instanceof Error && 'code' in error && error.code === 'EEXIST') { + refuseCustody('ALREADY_ENCRYPTED'); + } + throw error; + } + } finally { + rmSync(temporary, { force: true }); + } +} + /** * @typedef {{name: string, bytes: number, sha256: string, * recipient: {primary: string, subkey: string}, gnupg: string}} CustodyRecord @@ -181,136 +219,153 @@ function privateDirectory(path) { * }} options * @returns {Promise} */ -export async function encryptOnce({ - container, - recipient, - destination, - privateRoot, - now = Math.floor(Date.now() / 1000), - run = runCommand, - gpg, -}) { +export async function encryptOnce(options) { + /** @type {{work?: string}} */ + const scope = {}; + /** @type {{record?: CustodyRecord, error?: CustodyError}} */ + let outcome; + try { + outcome = { record: await encrypt(options, scope) }; + } catch (error) { + // A host error's message can name a private path: only fixed custody codes leave. + outcome = { error: error instanceof CustodyError ? error : new CustodyError('UNEXPECTED') }; + } + if (scope.work !== undefined) { + try { + rmSync(scope.work, { recursive: true, force: true }); + } catch { + // The plaintext may remain: refuse with a fixed code rather than a host message. + outcome = { error: new CustodyError('CLEANUP') }; + } + } + if (outcome.error) throw outcome.error; + return /** @type {CustodyRecord} */ (outcome.record); +} + +/** + * The encryption itself; `scope.work` names its work directory as soon as it exists. + * @param {Parameters[0]} options + * @param {{work?: string}} scope + * @returns {Promise} + */ +async function encrypt( + { + container, + recipient, + destination, + privateRoot, + now = Math.floor(Date.now() / 1000), + run = runCommand, + gpg, + }, + scope, +) { const pins = parsePins(recipient.pins); const { manifest } = readPrivateContainer(container); const name = ciphertextName(manifest.packageVersion); - const destinationInfo = lstatSync(destination); - if (!destinationInfo.isDirectory() || destinationInfo.isSymbolicLink()) { - refuseCustody('PLACEMENT'); - } + if (!plainDirectory(destination)) refuseCustody('PLACEMENT'); if (readdirSync(destination).includes(name)) refuseCustody('ALREADY_ENCRYPTED'); if (!privateDirectory(privateRoot)) refuseCustody('PRIVATE_ROOT'); const destinationPath = realpathSync(destination); const root = realpathSync(privateRoot); - if (within(root, destinationPath) || within(destinationPath, root)) refuseCustody('PLACEMENT'); + if (within(root, destinationPath) || within(destinationPath, root)) { + refuseCustody('PLACEMENT'); + } const binary = gpg ?? resolveGpg(); const work = mkdtempSync(join(root, 'custody-')); - try { - const env = gnupgEnvironment(work); - const listing = createGnupgHome(work, 'listing'); - const encryption = createGnupgHome(work, 'encryption'); - const recipientsHome = createGnupgHome(work, 'recipients'); - const call = async (args, code) => { - try { - return await run(binary.command, [...binary.prefix, ...args], { ...GNUPG_CALL, env }); - } catch { - return refuseCustody(code); - } - }; - const version = await gnupgVersion(binary, { run, env, home: encryption }); - - // Every later step reads this copy, so the bytes checked are the bytes encrypted to. - let key; + scope.work = work; + const env = gnupgEnvironment(work); + const listing = createGnupgHome(work, 'listing'); + const encryption = createGnupgHome(work, 'encryption'); + const recipientsHome = createGnupgHome(work, 'recipients'); + const call = async (args, code) => { try { - key = readOrdinaryFile(recipient.file, RECIPIENT_BYTES); + return await run(binary.command, [...binary.prefix, ...args], { ...GNUPG_CALL, env }); } catch { - refuseCustody('RECIPIENT_UNREADABLE'); + return refuseCustody(code); } - if (key.includes(SECRET_ARMOR)) refuseCustody('RECIPIENT_SECRET'); - const recipientPath = join(work, 'recipient.asc'); - writePrivateFile(recipientPath, key); - const listed = await call(listArgs(listing, recipientPath), 'RECIPIENT_UNREADABLE'); - inspectRecipient(parseColons(String(listed.stdout)), pins, now); - assertHomeState(listing, ['trustdb.gpg']); + }; + const version = await gnupgVersion(binary, { run, env, home: encryption }); - const input = join(work, 'container.tar'); - const output = join(work, 'ciphertext.gpg'); - writePrivateFile(input, container); - const encrypted = await call( - encryptArgs(encryption, recipientPath, input, output), - 'ENCRYPTION_FAILED', - ); - const status = parseStatus(String(encrypted.stdout)); - if (status.some((line) => FAILURE_STATUS.includes(line.keyword))) refuseCustody('STATUS'); - const begins = status.filter((line) => line.keyword === 'BEGIN_ENCRYPTION'); - const ends = status.filter((line) => line.keyword === 'END_ENCRYPTION'); - const considered = status.filter((line) => line.keyword === 'KEY_CONSIDERED'); - if ( - begins.length !== 1 || - ends.length !== 1 || - status.indexOf(ends[0]) < status.indexOf(begins[0]) || - considered.length !== 1 || - !isDeepStrictEqual(considered[0].args.slice(0, 2), [pins.primary, '0']) - ) { - refuseCustody('STATUS'); - } - // SEIPDv1 with MDC (method 2) and AES-256 (algorithm 9); an AEAD algorithm refuses. - if (begins[0].args[0] !== '2' || begins[0].args[1] !== '9' || begins[0].args.length > 2) { - refuseCustody('PROFILE'); - } - assertHomeState(encryption, ['random_seed']); + // Every later step reads this copy, so the bytes checked are the bytes encrypted to. + let key; + try { + key = readOrdinaryFile(recipient.file, RECIPIENT_BYTES); + } catch { + refuseCustody('RECIPIENT_UNREADABLE'); + } + if (key.includes(SECRET_ARMOR)) refuseCustody('RECIPIENT_SECRET'); + const recipientPath = join(work, 'recipient.asc'); + writePrivateFile(recipientPath, key); + const listed = await call(listArgs(listing, recipientPath), 'RECIPIENT_UNREADABLE'); + inspectRecipient(parseColons(String(listed.stdout)), pins, now); + assertHomeState(listing, ['trustdb.gpg']); - if ( - !isDeepStrictEqual(readdirSync(work).sort(), [ - 'ciphertext.gpg', - 'container.tar', - 'encryption', - 'listing', - 'recipient.asc', - 'recipients', - ]) - ) { - refuseCustody('ASSET'); - } - let ciphertext; - try { - ciphertext = readOrdinaryFile(output, container.length + CIPHERTEXT_OVERHEAD); - } catch { - refuseCustody('ASSET'); - } - const listedRecipients = await call(recipientsArgs(recipientsHome, output), 'RECIPIENTS'); - const recipients = parseStatus(String(listedRecipients.stdout)).filter( - (line) => line.keyword === 'ENC_TO', - ); - if ( - recipients.length !== 1 || - // A hidden recipient's all-zero key ID never equals the pinned subkey's. - recipients[0].args[0] !== pins.subkey.slice(-16) - ) { - refuseCustody('RECIPIENTS'); - } - assertHomeState(recipientsHome, []); + const input = join(work, 'container.tar'); + const output = join(work, 'ciphertext.gpg'); + writePrivateFile(input, container); + const encrypted = await call( + encryptArgs(encryption, recipientPath, input, output), + 'ENCRYPTION_FAILED', + ); + const status = parseStatus(String(encrypted.stdout)); + if (status.some((line) => FAILURE_STATUS.includes(line.keyword))) refuseCustody('STATUS'); + const begins = status.filter((line) => line.keyword === 'BEGIN_ENCRYPTION'); + const ends = status.filter((line) => line.keyword === 'END_ENCRYPTION'); + const considered = status.filter((line) => line.keyword === 'KEY_CONSIDERED'); + if ( + begins.length !== 1 || + ends.length !== 1 || + status.indexOf(ends[0]) < status.indexOf(begins[0]) || + considered.length !== 1 || + !isDeepStrictEqual(considered[0].args.slice(0, 2), [pins.primary, '0']) + ) { + refuseCustody('STATUS'); + } + // SEIPDv1 with MDC (method 2) and AES-256 (algorithm 9); an AEAD algorithm refuses. + if (begins[0].args[0] !== '2' || begins[0].args[1] !== '9' || begins[0].args.length > 2) { + refuseCustody('PROFILE'); + } + assertHomeState(encryption, ['random_seed']); - try { - writePrivateFile(join(destinationPath, name), ciphertext); - } catch (error) { - if (error instanceof Error && 'code' in error && error.code === 'EEXIST') { - refuseCustody('ALREADY_ENCRYPTED'); - } - throw error; - } - return Object.freeze({ - name, - bytes: ciphertext.length, - sha256: sha256(ciphertext), - recipient: Object.freeze({ primary: pins.primary, subkey: pins.subkey }), - gnupg: version, - }); - } catch (error) { - // A host error's message can name a private path: only fixed custody codes leave. - if (error instanceof CustodyError) throw error; - throw new CustodyError('UNEXPECTED'); - } finally { - rmSync(work, { recursive: true, force: true }); + if ( + !isDeepStrictEqual(readdirSync(work).sort(), [ + 'ciphertext.gpg', + 'container.tar', + 'encryption', + 'listing', + 'recipient.asc', + 'recipients', + ]) + ) { + refuseCustody('ASSET'); } + let ciphertext; + try { + ciphertext = readOrdinaryFile(output, container.length + CIPHERTEXT_OVERHEAD); + } catch { + refuseCustody('ASSET'); + } + const listedRecipients = await call(recipientsArgs(recipientsHome, output), 'RECIPIENTS'); + const recipients = parseStatus(String(listedRecipients.stdout)).filter( + (line) => line.keyword === 'ENC_TO', + ); + if ( + recipients.length !== 1 || + // A hidden recipient's all-zero key ID never equals the pinned subkey's. + recipients[0].args[0] !== pins.subkey.slice(-16) + ) { + refuseCustody('RECIPIENTS'); + } + assertHomeState(recipientsHome, []); + + publish(destinationPath, name, ciphertext); + return Object.freeze({ + name, + bytes: ciphertext.length, + sha256: sha256(ciphertext), + recipient: Object.freeze({ primary: pins.primary, subkey: pins.subkey }), + gnupg: version, + }); } diff --git a/tools/cli-package/custody.test.mjs b/tools/cli-package/custody.test.mjs index 9ee3f4e1..880ecc72 100644 --- a/tools/cli-package/custody.test.mjs +++ b/tools/cli-package/custody.test.mjs @@ -1,6 +1,7 @@ import assert from 'node:assert/strict'; import { spawn, spawnSync } from 'node:child_process'; import { Buffer } from 'node:buffer'; +import { createHash } from 'node:crypto'; import { appendFileSync, chmodSync, @@ -19,7 +20,14 @@ import { candidate, fresh, posixModes, seal } from './candidate-fixture.mjs'; import { buildPrivateContainer } from './container.mjs'; import { CustodyError } from './custody-error.mjs'; import { ciphertextName, encryptOnce, inspectRecipient, parsePins } from './custody.mjs'; -import { createGnupgHome, gnupgEnvironment, listArgs, parseColons, resolveGpg } from './gnupg.mjs'; +import { + createGnupgHome, + gnupgEnvironment, + gnupgVersion, + listArgs, + parseColons, + resolveGpg, +} from './gnupg.mjs'; import { PINS, ephemeralKey, fixture, processesNaming } from './gpg-fixture.mjs'; import { readPrivateManifest } from './manifest.mjs'; import { runCommand } from './process.mjs'; @@ -27,6 +35,14 @@ import { runCommand } from './process.mjs'; const refuses = (code) => (error) => error instanceof CustodyError && error.code === code; const NOW = Math.floor(Date.now() / 1000); +/** The installed GnuPG's version, read independently of `encryptOnce`. */ +async function installedVersion() { + const work = fresh('version'); + mkdirSync(work); + const home = createGnupgHome(work, 'home'); + return gnupgVersion(resolveGpg(), { run: runCommand, env: gnupgEnvironment(work), home }); +} + /** @type {Buffer} */ let container; /** @type {string} */ @@ -174,11 +190,19 @@ test('the recipient classification refuses in its fixed order', () => { ['RECIPIENT_UNUSABLE', records({ 11: 'cC' })], ['RECIPIENT_UNUSABLE', records({}, { 11: 's' })], ['RECIPIENT_UNUSABLE', records({}, { 1: 'n' })], + ['RECIPIENT_UNUSABLE', records({}, { 1: 'i' })], + ['RECIPIENT_UNUSABLE', records({}, { 11: 'eD' })], + ['RECIPIENT_UNUSABLE', records({ 1: 'n' })], + ['RECIPIENT_UNUSABLE', records({ 1: 'd' })], ['RECIPIENT_UNUSABLE', records({ 5: String(NOW + 60) })], ['RECIPIENT_UNUSABLE', records({}, { 5: String(NOW + 60) })], ['RECIPIENT_UNUSABLE', records({}, { 16: 'nistp256' })], ['RECIPIENT_UNUSABLE', records({}, { 3: '1', 2: '2048', 16: '' })], - ['RECIPIENT_UNUSABLE', records(), { primary: PRIMARY, subkey: PRIMARY }], + [ + 'RECIPIENT_UNUSABLE', + records({ 3: '18', 11: 'ecEC', 16: 'cv25519' }), + { primary: PRIMARY, subkey: PRIMARY }, + ], ['RECIPIENT_UNEXPECTED', [...records(), key('sub'), fpr('E'.repeat(40))]], ]) { assert.throws( @@ -266,11 +290,10 @@ test('encryption publishes exactly one ciphertext for the pinned subkey and leav assert.deepEqual(outcome.record, { name, bytes: bytes.length, - sha256: outcome.record.sha256, + sha256: createHash('sha256').update(bytes).digest('hex'), recipient: PINS.good, - gnupg: outcome.record.gnupg, + gnupg: await installedVersion(), }); - assert.match(outcome.record.sha256, /^[0-9a-f]{64}$/u); assert.ok(Object.isFrozen(outcome.record) && Object.isFrozen(outcome.record.recipient)); assert.deepEqual(calls, ['version', 'list', 'encrypt', 'recipients']); assert.deepEqual(readdirSync(outcome.root), []); @@ -319,11 +342,14 @@ test('every committed refusal fixture is refused before encryption, and AEAD aft refused(await attempt({ run, recipient: { file: fixture(name), pins: PINS[name] } }), code); assert.deepEqual(calls, ['version', 'list'], name); } - // GnuPG picks OCB for a key that advertises AEAD; that profile is refused. - refused( - await attempt({ recipient: { file: fixture('good-aead'), pins: PINS['good-aead'] } }), - 'PROFILE', - ); + // GnuPG 2.3 and later pick OCB for a key that advertises AEAD, a profile that is refused; GnuPG 2.2 + // has no AEAD and encrypts SEIPDv1, which the profile check admits. + const [major, minor] = (await installedVersion()).split('.').map(Number); + const aead = await attempt({ + recipient: { file: fixture('good-aead'), pins: PINS['good-aead'] }, + }); + if (major > 2 || (major === 2 && minor >= 3)) refused(aead, 'PROFILE'); + else assert.equal(aead.error, undefined, String(aead.error)); }); test("GnuPG's own refusal is a second layer behind the pre-check", async () => { @@ -355,6 +381,10 @@ test('the status must be exactly one pinned, completed SEIPDv1 AES-256 encryptio ], ['STATUS', (text) => `${text}[GNUPG:] INV_RECP 0 recipient\n`], ['STATUS', (text) => `${text}[GNUPG:] FAILURE encrypt 53\n`], + ...['NO_RECP 0', 'ERROR encrypt 1', 'KEYEXPIRED 1', 'KEYREVOKED', 'INV_SGNR 0 x'].map( + (line) => ['STATUS', (text) => `${text}[GNUPG:] ${line}\n`], + ), + ['STATUS', (text) => `${text}[GNUPG:] END_ENCRYPTION\n`], ['STATUS', (text) => text.replace(/\[GNUPG:\] KEY_CONSIDERED[^\n]*\n/u, '')], [ 'STATUS', @@ -365,6 +395,8 @@ test('the status must be exactly one pinned, completed SEIPDv1 AES-256 encryptio ['PROFILE', (text) => text.replace(/BEGIN_ENCRYPTION 2 9/u, 'BEGIN_ENCRYPTION 0 9 2')], ['PROFILE', (text) => text.replace(/BEGIN_ENCRYPTION 2 9/u, 'BEGIN_ENCRYPTION 2 7')], ['PROFILE', (text) => text.replace(/BEGIN_ENCRYPTION 2 9/u, 'BEGIN_ENCRYPTION 2 9 2')], + ['PROFILE', (text) => text.replace(/BEGIN_ENCRYPTION 2 9/u, 'BEGIN_ENCRYPTION 0 9')], + ['PROFILE', (text) => text.replace(/BEGIN_ENCRYPTION 2 9/u, 'BEGIN_ENCRYPTION 1 9')], ]) { refused(await attempt(recording(rewrite(change))), code); } @@ -464,6 +496,37 @@ test('a failing GnuPG call or an unreadable recipient refuses with a fixed code' refused(outcome, outcome.error.code); }); +test('the checked recipient copy is the one encrypted to, read within its bound', async () => { + // The caller's file changes after it was read: the checked copy is still the only key used. + const work = fresh('recipient-copy'); + mkdirSync(work); + const original = join(work, 'recipient.asc'); + writeFileSync(original, readFileSync(fixture('good'))); + const seen = []; + const swap = recording({ + list: { + before: (args) => { + seen.push(args.at(-1)); + writeFileSync(original, readFileSync(fixture('other'))); + }, + }, + encrypt: { + before: (args) => seen.push(args[args.indexOf('--recipient-file') + 1]), + }, + }); + const outcome = await attempt({ run: swap.run, recipient: { file: original, pins: PINS.good } }); + assert.equal(outcome.error, undefined, String(outcome.error)); + assert.deepEqual(outcome.record.recipient, PINS.good); + assert.equal(seen.length, 2); + for (const path of seen) { + assert.notEqual(path, original); + assert.equal(path.split(/[\\/]/u).at(-1), 'recipient.asc'); + } + const large = join(work, 'large.asc'); + writeFileSync(large, Buffer.concat([readFileSync(fixture('good')), Buffer.alloc(256 * 1024)])); + refused(await attempt({ recipient: { file: large, pins: PINS.good } }), 'RECIPIENT_UNREADABLE'); +}); + test('a ciphertext is published once: an existing one refuses before any GnuPG call', async () => { const { root, destination } = places(); writeFileSync(join(destination, NAME()), 'earlier ciphertext'); @@ -498,6 +561,7 @@ test('a ciphertext is published once: an existing one refuses before any GnuPG c refuses('ALREADY_ENCRYPTED'), ); assert.equal(readFileSync(join(raced.destination, NAME()), 'utf8'), 'raced ciphertext'); + assert.deepEqual(readdirSync(raced.destination), [NAME()]); assert.deepEqual(readdirSync(raced.root), []); }); @@ -547,6 +611,53 @@ test('the destination and the private root are separate directories, the root ow assert.deepEqual(calls, []); }); +test('a missing destination or private root refuses with a fixed code and no path', async () => { + const { root, destination } = places(); + const missing = join(fresh('missing'), 'nowhere'); + for (const [code, overrides] of [ + ['PLACEMENT', { destination: missing, privateRoot: root }], + ['PRIVATE_ROOT', { destination, privateRoot: missing }], + ]) { + await assert.rejects( + encryptOnce({ + container, + recipient: { file: fixture('good'), pins: PINS.good }, + ...overrides, + }), + (error) => + error instanceof CustodyError && + error.code === code && + !JSON.stringify({ message: error.message, ...error }).includes('nowhere'), + ); + } +}); + +test( + 'a work directory that cannot be removed refuses with a fixed code', + { skip: !posixModes || process.getuid?.() === 0 }, + async () => { + const { root, destination } = places(); + const locked = recording({ recipients: { before: () => chmodSync(root, 0o500) } }); + try { + await assert.rejects( + encryptOnce({ + container, + recipient: { file: fixture('good'), pins: PINS.good }, + destination, + privateRoot: root, + run: locked.run, + }), + (error) => + error instanceof CustodyError && + error.code === 'CLEANUP' && + !JSON.stringify({ message: error.message, ...error }).includes(root), + ); + } finally { + chmodSync(root, 0o700); + } + }, +); + test('no GnuPG output, path or host error text reaches a refusal', async () => { const marker = join(fresh('marker'), 'private', 'path'); const leaked = (error) => { @@ -649,6 +760,19 @@ test( assert.deepEqual(await closed, { code: 130, signal: null }); assert.equal(existsSync(root), false, 'the private root and its plaintext survived'); assert.deepEqual(readdirSync(destination), []); - assert.throws(() => process.kill(gpgPid, 0), { code: 'ESRCH' }); + // The group got SIGKILL before the driver exited; its member is reaped shortly after. + const alive = () => { + try { + process.kill(gpgPid, 0); + return true; + } catch (error) { + assert.equal(error.code, 'ESRCH'); + return false; + } + }; + for (let index = 0; index < 500 && alive(); index += 1) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + assert.equal(alive(), false, 'the stopped GnuPG stand-in outlived its group kill'); }, ); diff --git a/tools/cli-package/gnupg.mjs b/tools/cli-package/gnupg.mjs index 11031ad2..11a264fb 100644 --- a/tools/cli-package/gnupg.mjs +++ b/tools/cli-package/gnupg.mjs @@ -161,7 +161,7 @@ export function parseColons(text) { export async function gnupgVersion(gpg, { run, env, home }) { let stdout; try { - ({ stdout } = await run(gpg.command, [...gpg.prefix, '--homedir', home, '--version'], { + ({ stdout } = await run(gpg.command, [...gpg.prefix, ...isolated(home), '--version'], { ...GNUPG_CALL, env, })); diff --git a/tools/cli-package/gnupg.test.mjs b/tools/cli-package/gnupg.test.mjs index 6803f180..35a42028 100644 --- a/tools/cli-package/gnupg.test.mjs +++ b/tools/cli-package/gnupg.test.mjs @@ -63,6 +63,27 @@ test('the version floor is GnuPG 2.2.8, read from the version line only', async await assert.rejects(version(line), refuses('GPG_VERSION')); } await assert.rejects(version('gpg 2.4.4\n'), refuses('GPG_VERSION')); + // The version call is isolated like every other call. + let argv = []; + await gnupgVersion(gpg, { + run: async (_command, args) => { + argv = args; + return { stdout: 'gpg (GnuPG) 2.4.4\n', stderr: '' }; + }, + env: {}, + home: 'H', + }); + for (const flag of [ + '--homedir', + '--batch', + '--no-tty', + '--no-options', + '--no-autostart', + '--no-keyring', + ]) { + assert.ok(argv.includes(flag), flag); + } + assert.equal(argv.at(-1), '--version'); await assert.rejects( gnupgVersion(gpg, { run: async () => { diff --git a/tools/cli-package/ustar.test.mjs b/tools/cli-package/ustar.test.mjs index 2f9c1c37..e529c8d9 100644 --- a/tools/cli-package/ustar.test.mjs +++ b/tools/cli-package/ustar.test.mjs @@ -90,6 +90,11 @@ test('each header field outside the dialect refuses with its own code', () => { ['TAR_TYPE', mutated((block) => (block[156] = 0x35))], ['TAR_METADATA', mutated((block) => (block[157] = 0x61))], ['TAR_METADATA', mutated((block) => block.write('000001 \0', 108))], + ['TAR_METADATA', mutated((block) => block.write('000001 \0', 116))], + ['TAR_METADATA', mutated((block) => block.write('000001 \0', 329))], + ['TAR_METADATA', mutated((block) => block.write('000001 \0', 337))], + ['TAR_NUMBER', mutated((block) => block.write('0000000000x ', 136))], + ['TAR_FORMAT', mutated((block) => block.write('xx', 263))], ['TAR_NUMBER', mutated((block) => block.write('00000000009 ', 124))], ['TAR_NAME', mutated((block) => (block[20] = 0x61))], ['UTF8', mutated((block) => block.fill(0xff, 0, 4))], From dea32ea61a0aed3eeb66875623752a80ecdd92e6 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 12:22:50 +0300 Subject: [PATCH 72/97] fix(repo): address the first Step 9b.1 review round Round 1 at f0555a61 reported three medium findings, all corrected: - GnuPG reads its installation's /gpg.conf under --no-options, and a global log-file with debug options could write the session key outside every home; the installation's gpgconf now names that directory before any key is handled, and a gpg.conf or common.conf there refuses (GPG_CONFIG); - buildPrivateContainer let host errors carrying the staging path escape; they now refuse CONTAINER_UNREADABLE; - the algorithm-ID half of the key policy had no test; ElGamal cases refuse. Lows corrected: the destination/root overlap is compared by device and inode; the listed ciphertext file must still be the published bytes; the plaintext work directory is removed before publishing, so CLEANUP publishes nothing; tests for revocation before expiry, an expired second subkey, ssb-only secrets, the default clock unit, record item keys, the isolated environment, owner-only homes and the --ignore-* family; and wording in the fixture comment, the contract, the Wave 4b clause (AES256 preference) and the checkpoint. Two lows are recorded in deferred-tasks.md. Refs: ADR-0111, Step 9b.1 Co-Authored-By: Claude Opus 5.5 --- docs/reference/cli/packaging.md | 27 +++- docs/roadmap/deferred-tasks.md | 15 +- .../phase-2.5.5-hardening-and-remediation.md | 35 ++++- tools/cli-package/container.mjs | 13 +- tools/cli-package/container.test.mjs | 24 +++ tools/cli-package/custody.mjs | 46 +++++- tools/cli-package/custody.test.mjs | 145 ++++++++++++++++-- tools/cli-package/gnupg.mjs | 60 +++++++- tools/cli-package/gnupg.test.mjs | 40 ++++- tools/cli-package/gpg-fixture.mjs | 7 +- tools/cli-package/ustar.test.mjs | 4 +- 11 files changed, 368 insertions(+), 48 deletions(-) diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 342c9195..04da0815 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -536,9 +536,14 @@ recorded in the [Wave 3 register](../../roadmap/phases/phase-2.5.5-hardening-and as a JSON object. Reading a container back applies the same checks, and a decrypted container extracted with `tar -x` is a staging the manifest reader consumes unchanged. - **The isolated runner.** `gnupg.mjs` resolves `gpg` from absolute PATH entries only and - refuses one older than GnuPG 2.2.8. Every call runs in a fresh owner-only home the tool - creates, with `--no-options --no-autostart --no-keyring`, the C locale and an isolated - environment, bounded by `runCommand`. The recipient listing (`--show-keys`) uses its own home + refuses one older than GnuPG 2.2.8; the installation — its binary, its `gpgconf` and its + system-wide configuration directory — is trusted. Every call runs in a fresh owner-only home + the tool creates, with `--no-options --no-autostart --no-keyring`, the C locale and an + isolated environment, bounded by `runCommand`. GnuPG still reads the installation's + `/gpg.conf` under `--no-options`, and its `[force]` options override the command + line, so before any key is handled the installation's `gpgconf --list-dirs sysconfdir` is + read and a `gpg.conf` or `common.conf` there refuses (`GPG_CONFIG`): a global configuration + could add or hide a recipient, redirect the output or log the session key. The recipient listing (`--show-keys`) uses its own home and the default trust model, because GnuPG 2.4 cannot list an expired or revoked key under `--trust-model always` without a trust database; that home may gain only GnuPG's own `trustdb.gpg`. Encryption uses `--recipient-file`, `--trust-model always` (the pinned @@ -546,7 +551,9 @@ recorded in the [Wave 3 register](../../roadmap/phases/phase-2.5.5-hardening-and compression and a fixed literal name, in a home that may gain only `random_seed`. - **The recipient pre-check.** The recipient file is read bounded and copied, so the bytes checked are the bytes encrypted to, and pins are exactly two 40-digit uppercase fingerprints - (OpenPGP v4). An armored secret key refuses from its bytes. The colon listing then refuses, + (OpenPGP v4). An armored secret key refuses from its bytes; other secret material refuses + from the listing as secret, or as unreadable where GnuPG cannot list it. The colon listing + then refuses, in this order: secret material, a missing pinned primary, more than one listed key, a missing pinned subkey, revocation, expiry at the current time, unusability (invalid or disabled, no encryption capability, a creation time in the future, or an encryption subkey other than @@ -556,11 +563,14 @@ recorded in the [Wave 3 register](../../roadmap/phases/phase-2.5.5-hardening-and `KEY_CONSIDERED` and recipient checks remain the independent second layer for it. - **Encrypt once.** `encryptOnce` refuses before any GnuPG call when the destination already holds `relavium--private-maps.tar.gpg`, when the destination and the owner-only - private root overlap, or when the root is not owner-only. Success requires exit 0, one + private root overlap (compared by device and inode, so a second spelling of one directory + does not hide the overlap), or when either is missing or the root is not owner-only. Success requires exit 0, one `KEY_CONSIDERED` for the pinned primary, one `BEGIN_ENCRYPTION 2 9` (SEIPDv1 with MDC and AES-256; an AEAD profile refuses) before one `END_ENCRYPTION`, exactly the expected work files, a ciphertext within the container's size plus 64 KiB whose public-key packets name - exactly the pinned subkey and no hidden recipient, and homes holding only what GnuPG creates. + exactly the pinned subkey and no hidden recipient and whose file is still the bytes about to + be published, and homes holding only what GnuPG creates. The work directory and its + plaintext are removed before anything is published, so a failed removal publishes nothing. The ciphertext is then published atomically: written to a fresh sibling, then hard-linked to its final name, which never replaces an existing file, so a failed write never leaves a partial file under that name. It returns a frozen custody record: name, size, SHA-256, pins @@ -569,8 +579,9 @@ recorded in the [Wave 3 register](../../roadmap/phases/phase-2.5.5-hardening-and an owner-only work directory under the caller's private root, removed on every path; an interrupt in the caller's `exitOnInterrupt` scope stops GnuPG and removes the root. No secure erasure is claimed. Every refusal is a fixed `PACKED_CLI_CUSTODY_`, including a host - failure (`UNEXPECTED`) and a work directory that cannot be removed (`CLEANUP`); GnuPG's stdout - and stderr, paths and host error text never surface. + failure (`UNEXPECTED`, or `CONTAINER_UNREADABLE` for a staging that cannot be read) and a + work directory that cannot be removed (`CLEANUP`); GnuPG's stdout and stderr, paths and host + error text never surface. - **Synthetic fixtures.** `custody-fixtures/` commits armored public v4 keys under `fixture.invalid` for the classes GnuPG can produce — good cv25519 and RSA-3072 keys, one advertising AEAD, an RSA-2048 subkey, an expired primary and subkey, a revoked primary and diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index 073f148c..8f8cd62c 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -805,6 +805,17 @@ Recorded with the Step 9b.1 implementation outcome, so nothing is published. **Re-check:** count primary-key packets independently of import filtering if a recipient file is ever produced by anything but `gpg --export`. *(low · tools/cli-package/custody.mjs)* +- [ ] **Secret material GnuPG neither lists nor reads is admitted.** *Low; round 1, R1-COR-2.* A binary + secret key appended after the armored public block is ignored by GnuPG's listing and encryption, so the + pre-check admits the file and the ciphertext is published. The secret stays in the recipient file and + never reaches the ciphertext, the record or a log. **Re-check:** refuse any byte after the armored public + block's end line (or any secret-key packet tag in a packet walk) if recipient files are ever produced by + anything but `gpg --armor --export`. *(low · tools/cli-package/custody.mjs)* +- [ ] **The atomic publish has no fault-injection test.** *Low; round 1, T-9B1-R1-3.* `publish` writes a + temporary sibling and hard-links it to the final name; writing the final name directly also passes every + test, because no test makes the write fail midway. **Re-check:** inject a write failure (a full + filesystem or a writer port) and assert no file exists under the final name afterwards. *(low · + tools/cli-package/custody.mjs)* - [ ] **The GnuPG 2.2.8 floor is enforced, never exercised.** *Low; a named limit.* The version check refuses an older GnuPG, but the tests ran only on 2.5.21 (local) and 2.4.4 (Ubuntu 24.04), so neither encryption nor the `gpg -d | tar -x` recovery is proven on 2.2.8. **Re-check:** run the custody tests @@ -2689,7 +2700,9 @@ future test cannot silently re-acquire it. must not advertise the AEAD feature, because custody requires SEIPDv1 with MDC and AES-256 so that `gpg -d | tar -x` with GnuPG ≥ 2.2.8 stays true; and the committed recipient file is the export of the primary with only the pinned encryption subkey (`gpg --armor --export '!'`), - because a second usable encryption subkey refuses. + because a second usable encryption subkey refuses. *Added after Step 9b.1's review round 1:* its + cipher preferences must include AES256, since custody passes an AES-256 preference and refuses any + other cipher. - [ ] **Scheduled → Wave 5a: alt-screen word wrapping (#61).** The maintainer approved retaining character-wrap replacement with render-v2/D18 and picker diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index a58e409e..e2334b15 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -10102,8 +10102,9 @@ and this record: 2. **The private container.** `container.mjs` builds and reads the owner-only container in the approved staging-relative layout, every map checked against the manifest and the finalizer's record. 3. **The isolated GnuPG runner.** `gnupg.mjs`, with GnuPG ≥ 2.2.8 added to the local prerequisites. -4. **Synthetic fixtures.** Committed armored public keys for every recipient class, their pins and a - maintainer-only generator; the round-trip key is generated per run. +4. **Synthetic fixtures.** Committed armored public keys for the recipient classes GnuPG can produce, + their pins and a maintainer-only generator; crafted listings cover the other classes, and the round-trip key + is generated per run. 5. **Encrypt once.** `custody.mjs`: pins, the colon pre-check, `encryptOnce` and its tests. The [packaging contract](../../reference/cli/packaging.md#recipient-custody-implementation-boundary--step-9b1) records @@ -10172,3 +10173,33 @@ inventory's `vitest list` collection exceeded its 30-second spawn timeout while load average at 40–66. On a quiet machine it passed three of three runs in 10–12 seconds. The full CI reruns at the corrected head, alone, before the first round. +**Author evidence at `f0555a61`.** The full `pnpm run ci` passed alone (exit 0): `test:acceptance-tools` 969 of +969, all 25 product map controls, and every causal-control family complete. + +**Round 1 at `f0555a61` — NONCLEAN.** Five lenses and a critic ran, every finding adversarially verified. They +reported three medium findings, all corrected: +- **Global configuration.** GnuPG reads its installation's `/gpg.conf` even under `--no-options`, so + the "no options file" claim was false. The reviewer showed that a global `log-file` with debug options wrote + the AES session key outside every home while the encryption passed every check. The installation's `gpgconf` + now names that directory before any key is handled, and a `gpg.conf` or `common.conf` there refuses + (`GPG_CONFIG`). The contract names the installation as trusted. +- **Container host errors.** `buildPrivateContainer` still let host errors carrying the staging path escape; the + rigor pass had named it with `encryptOnce`, and only `encryptOnce` was corrected. They now refuse + `CONTAINER_UNREADABLE`. +- **Algorithm policy.** Nothing tested the algorithm-ID half of the policy: an ElGamal subkey would pass once + that half was removed. ElGamal cases now refuse. + +Lows corrected: +- **Overlap.** The destination and root are compared by device and inode, not spelling; a case-variant test runs + on case-insensitive volumes and is skipped where the volume is case-sensitive. +- **Publish order.** The listed ciphertext file must still be the bytes published, and the plaintext work + directory is removed before anything is published, so a `CLEANUP` refusal publishes nothing. +- **Tests.** New cases cover revocation before expiry, an expired second subkey, `ssb`-only secret material, the + default clock's unit, the record items' exact keys, the isolated environment, the C locale and owner-only homes, + and the forbidden `--ignore-*` family. +- **Wording.** Corrected in the fixture comment, the contract's secret-material wording, the Wave 4b clause (an + AES256 preference) and this checkpoint's fixture item. + +Two lows are recorded in deferred-tasks.md: secret material GnuPG neither lists nor reads is admitted, and the +atomic publish has no fault-injection test. Every corrected guard's mutant now fails a test. + diff --git a/tools/cli-package/container.mjs b/tools/cli-package/container.mjs index a8e864b5..463c6ead 100644 --- a/tools/cli-package/container.mjs +++ b/tools/cli-package/container.mjs @@ -10,7 +10,7 @@ import { Buffer } from 'node:buffer'; import { lstatSync, readdirSync } from 'node:fs'; import { join, posix } from 'node:path'; import { isDeepStrictEqual } from 'node:util'; -import { refuseCustody } from './custody-error.mjs'; +import { CustodyError, refuseCustody } from './custody-error.mjs'; import { readOrdinaryFile, sha256 } from './files.mjs'; import { MANIFEST_FILE, validateManifest } from './manifest.mjs'; import { readUstar, writeUstar } from './ustar.mjs'; @@ -147,6 +147,17 @@ function containerBytes(files) { * @returns {{bytes: Buffer, manifest: import('./manifest.mjs').PrivateManifest}} */ export function buildPrivateContainer(staging, limits = CONTAINER_LIMITS) { + try { + return build(staging, limits); + } catch (error) { + // A host error's message names the staging path: only fixed custody codes leave. + if (error instanceof CustodyError) throw error; + return refuseCustody('CONTAINER_UNREADABLE'); + } +} + +/** @param {string} staging @param {typeof CONTAINER_LIMITS} limits */ +function build(staging, limits) { /** @param {string} path */ const read = (path) => { try { diff --git a/tools/cli-package/container.test.mjs b/tools/cli-package/container.test.mjs index 114ac2b3..88d60c0f 100644 --- a/tools/cli-package/container.test.mjs +++ b/tools/cli-package/container.test.mjs @@ -177,6 +177,15 @@ test('a staging that is not exactly the sealed records and maps refuses', () => writeFileSync(path, JSON.stringify(record)); }, ], + [ + 'CONTAINER_RECORD', + (copy) => { + const path = join(copy, 'finalized.json'); + const record = JSON.parse(readFileSync(path, 'utf8')); + record.maps[0].extra = true; + writeFileSync(path, JSON.stringify(record)); + }, + ], ['CONTAINER_RECORD', (copy) => writeFileSync(join(copy, 'metafile-esm.json'), 'not json')], ['CONTAINER_RECORD', (copy) => writeFileSync(join(copy, 'metafile-esm.json'), '[]')], ]) { @@ -184,6 +193,21 @@ test('a staging that is not exactly the sealed records and maps refuses', () => } }); +test('a staging that cannot be read refuses with a fixed code and no path', () => { + const missing = join(fresh('no-staging'), 'staging'); + const file = `${fresh('file-staging')}.json`; + writeFileSync(file, '{}'); + for (const path of [missing, file]) { + assert.throws( + () => buildPrivateContainer(path), + (error) => + error instanceof CustodyError && + error.code === 'CONTAINER_UNREADABLE' && + !error.message.includes(path), + ); + } +}); + test('a linked record or maps directory refuses', { skip: !posixModes }, () => { assert.throws( () => diff --git a/tools/cli-package/custody.mjs b/tools/cli-package/custody.mjs index 4fa8b77b..6034494b 100644 --- a/tools/cli-package/custody.mjs +++ b/tools/cli-package/custody.mjs @@ -10,8 +10,16 @@ * production recipient: Step 9b uses synthetic recipients only. */ import { randomUUID } from 'node:crypto'; -import { linkSync, lstatSync, mkdtempSync, readdirSync, realpathSync, rmSync } from 'node:fs'; -import { isAbsolute, join, relative, sep } from 'node:path'; +import { + linkSync, + lstatSync, + mkdtempSync, + readdirSync, + realpathSync, + rmSync, + statSync, +} from 'node:fs'; +import { dirname, join } from 'node:path'; import { isDeepStrictEqual } from 'node:util'; import { readPrivateContainer } from './container.mjs'; import { CustodyError, refuseCustody } from './custody-error.mjs'; @@ -20,6 +28,7 @@ import { writePrivateFile } from './finalize.mjs'; import { GNUPG_CALL, assertHomeState, + assertNoGlobalConfig, createGnupgHome, encryptArgs, gnupgEnvironment, @@ -33,7 +42,10 @@ import { import { runCommand } from './process.mjs'; const FINGERPRINT = /^[0-9A-F]{40}$/u; -/** An armored secret key: refused from the bytes, since listing one needs an agent this boundary never starts. */ +/** + * An armored secret key, refused from the bytes before any listing: whether GnuPG lists secret input + * (as `sec`/`ssb`, refused too) or fails depends on the home's agent socket path. + */ const SECRET_ARMOR = ['-----BEGIN PGP', 'PRIVATE KEY BLOCK-----'].join(' '); const RECIPIENT_BYTES = 256 * 1024; /** Literal-packet and packet framing overhead a ciphertext may add to its container. */ @@ -148,8 +160,14 @@ export function inspectRecipient(records, pins, now) { /** @param {string} root @param {string} path */ function within(root, path) { - const rel = relative(root, path); - return rel === '' || (!isAbsolute(rel) && rel !== '..' && !rel.startsWith(`..${sep}`)); + // Compared by identity, not spelling: a case-insensitive volume or a bind mount can give one + // directory two spellings that no string comparison relates. + const { dev, ino } = statSync(root, { bigint: true }); + for (let current = path; ; current = dirname(current)) { + const info = statSync(current, { bigint: true }); + if (info.dev === dev && info.ino === ino) return true; + if (dirname(current) === current) return false; + } } /** A directory, not a link; a missing path is not one. */ @@ -215,7 +233,7 @@ function publish(directory, name, bytes) { * privateRoot: string, * now?: number, * run?: typeof runCommand, - * gpg?: {command: string, prefix: string[]}, + * gpg?: import('./gnupg.mjs').GnupgInstallation, * }} options * @returns {Promise} */ @@ -287,6 +305,7 @@ async function encrypt( } }; const version = await gnupgVersion(binary, { run, env, home: encryption }); + await assertNoGlobalConfig(binary, { run, env }); // Every later step reads this copy, so the bytes checked are the bytes encrypted to. let key; @@ -359,7 +378,22 @@ async function encrypt( refuseCustody('RECIPIENTS'); } assertHomeState(recipientsHome, []); + // The listed file must still be the bytes about to be published. + let listedBytes; + try { + listedBytes = readOrdinaryFile(output, container.length + CIPHERTEXT_OVERHEAD); + } catch { + refuseCustody('ASSET'); + } + if (!listedBytes.equals(ciphertext)) refuseCustody('ASSET'); + // The plaintext goes before anything is published, so a failed removal publishes nothing. + try { + rmSync(work, { recursive: true, force: true }); + } catch { + refuseCustody('CLEANUP'); + } + scope.work = undefined; publish(destinationPath, name, ciphertext); return Object.freeze({ name, diff --git a/tools/cli-package/custody.test.mjs b/tools/cli-package/custody.test.mjs index 880ecc72..94a4a371 100644 --- a/tools/cli-package/custody.test.mjs +++ b/tools/cli-package/custody.test.mjs @@ -69,15 +69,17 @@ function places() { /** @param {string[]} args */ const kindOf = (args) => - args.includes('--version') - ? 'version' - : args.includes('--show-keys') - ? 'list' - : args.includes('--encrypt') - ? 'encrypt' - : args.includes('--list-only') - ? 'recipients' - : 'other'; + args.includes('--list-dirs') + ? 'config' + : args.includes('--version') + ? 'version' + : args.includes('--show-keys') + ? 'list' + : args.includes('--encrypt') + ? 'encrypt' + : args.includes('--list-only') + ? 'recipients' + : 'other'; /** @param {string[]} args @param {string} flag */ const valueOf = (args, flag) => args[args.indexOf(flag) + 1]; @@ -198,6 +200,12 @@ test('the recipient classification refuses in its fixed order', () => { ['RECIPIENT_UNUSABLE', records({}, { 5: String(NOW + 60) })], ['RECIPIENT_UNUSABLE', records({}, { 16: 'nistp256' })], ['RECIPIENT_UNUSABLE', records({}, { 3: '1', 2: '2048', 16: '' })], + // An ElGamal subkey of any size is not an approved algorithm. + ['RECIPIENT_UNUSABLE', records({}, { 3: '16', 2: '4096', 16: '' })], + ['RECIPIENT_UNUSABLE', records({}, { 3: '16', 2: '255', 16: 'cv25519' })], + // Revocation is reported before expiry. + ['RECIPIENT_REVOKED', records({ 1: 'r', 6: String(NOW) })], + ['RECIPIENT_SECRET', [...records(), key('ssb'), fpr('E'.repeat(40))]], [ 'RECIPIENT_UNUSABLE', records({ 3: '18', 11: 'ecEC', 16: 'cv25519' }), @@ -217,6 +225,19 @@ test('the recipient classification refuses in its fixed order', () => { inspectRecipient([...records(), key('sub', { 1: 'r' }), fpr('E'.repeat(40))], pins, NOW), pins, ); + assert.equal( + inspectRecipient([...records(), key('sub', { 1: 'e' }), fpr('E'.repeat(40))], pins, NOW), + pins, + ); + assert.equal( + inspectRecipient( + [...records(), key('sub', { 6: String(NOW) }), fpr('E'.repeat(40))], + pins, + NOW, + ), + pins, + ); + assert.equal(inspectRecipient(records({}, { 3: '1', 2: '4096', 16: '' }), pins, NOW), pins); }); test('the committed synthetic fixtures list as their names say under the real GnuPG', async () => { @@ -295,7 +316,7 @@ test('encryption publishes exactly one ciphertext for the pinned subkey and leav gnupg: await installedVersion(), }); assert.ok(Object.isFrozen(outcome.record) && Object.isFrozen(outcome.record.recipient)); - assert.deepEqual(calls, ['version', 'list', 'encrypt', 'recipients']); + assert.deepEqual(calls, ['version', 'config', 'list', 'encrypt', 'recipients']); assert.deepEqual(readdirSync(outcome.root), []); }); @@ -340,7 +361,7 @@ test('every committed refusal fixture is refused before encryption, and AEAD aft ]) { const { run, calls } = recording(); refused(await attempt({ run, recipient: { file: fixture(name), pins: PINS[name] } }), code); - assert.deepEqual(calls, ['version', 'list'], name); + assert.deepEqual(calls, ['version', 'config', 'list'], name); } // GnuPG 2.3 and later pick OCB for a key that advertises AEAD, a profile that is refused; GnuPG 2.2 // has no AEAD and encrypts SEIPDv1, which the profile check admits. @@ -363,7 +384,7 @@ test("GnuPG's own refusal is a second layer behind the pre-check", async () => { await attempt({ run, recipient: { file: fixture('expired-subkey'), pins } }), 'ENCRYPTION_FAILED', ); - assert.deepEqual(calls, ['version', 'list', 'encrypt']); + assert.deepEqual(calls, ['version', 'config', 'list', 'encrypt']); }); test('the status must be exactly one pinned, completed SEIPDv1 AES-256 encryption', async () => { @@ -527,6 +548,72 @@ test('the checked recipient copy is the one encrypted to, read within its bound' refused(await attempt({ recipient: { file: large, pins: PINS.good } }), 'RECIPIENT_UNREADABLE'); }); +test("a global GnuPG configuration in the installation's sysconfdir refuses before any key is handled", async () => { + const sysconf = fresh('sysconfdir'); + mkdirSync(sysconf); + const config = (stdout) => ({ config: { replace: () => ({ stdout, stderr: '' }) } }); + // An empty configuration directory, as Homebrew and Ubuntu ship it, admits. + const admitted = recording(config(`${sysconf}\n`)); + const outcome = await attempt(admitted); + assert.equal(outcome.error, undefined, String(outcome.error)); + for (const name of ['gpg.conf', 'common.conf']) { + const directory = fresh(`sysconf-${name}`); + mkdirSync(directory); + writeFileSync(join(directory, name), 'log-file /tmp/leak.log\ndebug-all\n'); + const { run, calls } = recording(config(`${directory}\n`)); + refused(await attempt({ run }), 'GPG_CONFIG'); + assert.deepEqual(calls, ['version', 'config'], name); + } + // gpgconf percent-escapes a directory name; a relative or malformed answer refuses. + const escaped = fresh('sys conf'); + mkdirSync(escaped); + writeFileSync(join(escaped, 'gpg.conf'), 'throw-keyids\n'); + refused(await attempt(recording(config(`${escaped.replaceAll(' ', '%20')}\n`))), 'GPG_CONFIG'); + refused(await attempt(recording(config('relative/dir\n'))), 'GPG_CONFIG'); + refused(await attempt(recording(config('%zz\n'))), 'GPG_CONFIG'); + refused( + await attempt( + recording({ + config: { + replace: () => { + throw new Error('gpgconf failed'); + }, + }, + }), + ), + 'GPG_UNAVAILABLE', + ); +}); + +test('the published bytes are the listed bytes, and the default clock counts seconds', async () => { + // A ciphertext file replaced after it was read — here by another valid ciphertext for the same + // recipient, so its listing still passes — is refused, never published. + const earlier = await attempt(); + assert.equal(earlier.error, undefined, String(earlier.error)); + const other = readFileSync(join(earlier.destination, NAME())); + const swapped = recording({ + recipients: { before: (args) => writeFileSync(args.at(-1), other) }, + }); + refused(await attempt(swapped), 'ASSET'); + // A recipient whose subkey expires tomorrow is valid under the default clock (seconds since the + // epoch); a clock in milliseconds would call it expired. + const pins = PINS.good; + const tomorrow = String(Math.floor(Date.now() / 1000) + 86_400); + const listing = [key('pub'), fpr(pins.primary), key('sub', { 6: tomorrow }), fpr(pins.subkey)] + .map((fields) => fields.join(':')) + .join('\n'); + const { root, destination } = places(); + const forged = recording({ list: { replace: () => ({ stdout: listing, stderr: '' }) } }); + const record = await encryptOnce({ + container, + recipient: { file: fixture('good'), pins }, + destination, + privateRoot: root, + run: forged.run, + }); + assert.equal(record.name, NAME()); +}); + test('a ciphertext is published once: an existing one refuses before any GnuPG call', async () => { const { root, destination } = places(); writeFileSync(join(destination, NAME()), 'earlier ciphertext'); @@ -611,6 +698,30 @@ test('the destination and the private root are separate directories, the root ow assert.deepEqual(calls, []); }); +test('an overlap spelled differently on a case-insensitive volume still refuses', async (context) => { + const base = fresh('case'); + mkdirSync(join(base, 'Public'), { recursive: true }); + if (!existsSync(join(base, 'PUBLIC'))) { + context.skip('this volume is case-sensitive'); + return; + } + const root = join(base, 'PUBLIC', 'root'); + mkdirSync(root); + chmodSync(root, 0o700); + const { run, calls } = recording(); + await assert.rejects( + encryptOnce({ + container, + recipient: { file: fixture('good'), pins: PINS.good }, + destination: join(base, 'Public'), + privateRoot: root, + run, + }), + refuses('PLACEMENT'), + ); + assert.deepEqual(calls, []); +}); + test('a missing destination or private root refuses with a fixed code and no path', async () => { const { root, destination } = places(); const missing = join(fresh('missing'), 'nowhere'); @@ -652,6 +763,8 @@ test( error.code === 'CLEANUP' && !JSON.stringify({ message: error.message, ...error }).includes(root), ); + // The plaintext is removed before anything is published. + assert.deepEqual(readdirSync(destination), []); } finally { chmodSync(root, 0o700); } @@ -698,7 +811,7 @@ test('no GnuPG output, path or host error text reaches a refusal', async () => { return error instanceof CustodyError && error.code === 'UNEXPECTED'; }, ); - assert.deepEqual(vanished.calls, ['version', 'list', 'encrypt', 'recipients']); + assert.deepEqual(vanished.calls, ['version', 'config', 'list', 'encrypt', 'recipients']); assert.deepEqual(readdirSync(lost.root), []); }); @@ -725,7 +838,8 @@ test( [ "import { readFileSync, writeFileSync } from 'node:fs';", 'const args = process.argv.slice(2);', - "if (args.includes('--version')) process.stdout.write('gpg (GnuPG) 2.4.4\\n');", + `if (args.includes('--list-dirs')) process.stdout.write(${JSON.stringify(work)} + '\\n');`, + "else if (args.includes('--version')) process.stdout.write('gpg (GnuPG) 2.4.4\\n');", `else if (args.includes('--show-keys')) process.stdout.write(readFileSync(${JSON.stringify(listing)}, 'utf8'));`, `else { writeFileSync(${JSON.stringify(pidFile)}, String(process.pid)); writeFileSync(${JSON.stringify(started)}, ''); setInterval(() => {}, 1000); }`, ].join('\n'), @@ -743,7 +857,8 @@ test( ` encryptOnce({ container: readFileSync(${JSON.stringify(join(work, 'container.tar'))}),`, ` recipient: { file: ${JSON.stringify(fixture('good'))}, pins: ${JSON.stringify(pins)} },`, ` destination: ${JSON.stringify(destination)}, privateRoot: root,`, - ` gpg: { command: process.execPath, prefix: [${JSON.stringify(fakeGpg)}] } }));`, + ` gpg: { command: process.execPath, prefix: [${JSON.stringify(fakeGpg)}],`, + ` gpgconf: { command: process.execPath, prefix: [${JSON.stringify(fakeGpg)}] } } }));`, ].join('\n'), ); writeFileSync(join(work, 'container.tar'), container); diff --git a/tools/cli-package/gnupg.mjs b/tools/cli-package/gnupg.mjs index 11a264fb..757ae5fa 100644 --- a/tools/cli-package/gnupg.mjs +++ b/tools/cli-package/gnupg.mjs @@ -2,11 +2,13 @@ * The isolated GnuPG boundary of the custody tools (Step 9b.1; packaging.md, "Recipient custody"). * GnuPG is release tooling, never a runtime dependency. Every call runs the resolved absolute binary * through the shell-free, bounded `runCommand`, in a fresh owner-only home the tools create themselves, - * with no options file, keyring, agent or key discovery. Only status lines and colon records are - * parsed; human-readable output and stderr are never read or surfaced. + * with no per-home options file, keyring, agent or key discovery. GnuPG still reads its installation's + * system-wide `gpg.conf` under `--no-options`, so a global configuration refuses before any call that + * handles a key. Only status lines, colon records, the version line and the configuration directory + * are read; other human-readable output and stderr are never read or surfaced. */ import { existsSync, lstatSync, mkdirSync, readdirSync, realpathSync, statSync } from 'node:fs'; -import { delimiter, isAbsolute, join } from 'node:path'; +import { delimiter, dirname, isAbsolute, join } from 'node:path'; import { refuseCustody } from './custody-error.mjs'; import { isolatedEnvironment } from './process.mjs'; @@ -18,15 +20,27 @@ export const GNUPG_CALL = Object.freeze({ timeoutMs: 60_000, maxBytes: 256 * 102 export const LITERAL_NAME = 'relavium-private-maps.tar'; /** - * Resolve `gpg` from absolute PATH entries only, without a shell. - * @returns {{command: string, prefix: string[]}} + * @typedef {{command: string, prefix: string[]}} Command + * @typedef {Command & {gpgconf: Command}} GnupgInstallation + */ + +/** + * Resolve `gpg` from absolute PATH entries only, without a shell, and the `gpgconf` of the same + * installation beside it. + * @returns {GnupgInstallation} */ export function resolveGpg(path = process.env.PATH ?? '', platform = process.platform) { + const executable = (name) => (platform === 'win32' ? `${name}.exe` : name); for (const directory of path.split(delimiter).filter(Boolean)) { if (!isAbsolute(directory)) continue; - const candidate = join(directory, platform === 'win32' ? 'gpg.exe' : 'gpg'); + const candidate = join(directory, executable('gpg')); if (existsSync(candidate) && statSync(candidate).isFile()) { - return { command: realpathSync(candidate), prefix: [] }; + const command = realpathSync(candidate); + return { + command, + prefix: [], + gpgconf: { command: join(dirname(command), executable('gpgconf')), prefix: [] }, + }; } } return refuseCustody('GPG_UNAVAILABLE'); @@ -178,6 +192,38 @@ export async function gnupgVersion(gpg, { run, env, home }) { return version.join('.'); } +/** + * GnuPG reads `/gpg.conf` even under `--no-options`, and `[force]` options there override + * the command line: a global configuration could add or hide a recipient, redirect the output or log + * the session key. The installation's `gpgconf` names that directory; a `gpg.conf` or `common.conf` + * in it refuses. + * @param {GnupgInstallation} gpg + * @param {{run: Function, env: Record}} options + */ +export async function assertNoGlobalConfig(gpg, { run, env }) { + let stdout; + try { + ({ stdout } = await run( + gpg.gpgconf.command, + [...gpg.gpgconf.prefix, '--list-dirs', 'sysconfdir'], + { ...GNUPG_CALL, env }, + )); + } catch { + return refuseCustody('GPG_UNAVAILABLE'); + } + let directory; + try { + // gpgconf percent-escapes special characters in the directories it lists. + directory = decodeURIComponent(String(stdout).trim()); + } catch { + return refuseCustody('GPG_CONFIG'); + } + if (!isAbsolute(directory)) refuseCustody('GPG_CONFIG'); + for (const name of ['gpg.conf', 'common.conf']) { + if (existsSync(join(directory, name))) refuseCustody('GPG_CONFIG'); + } +} + /** * The home may hold only regular files GnuPG itself creates for these calls: never a socket, a * configuration, a keyring or private-key material. diff --git a/tools/cli-package/gnupg.test.mjs b/tools/cli-package/gnupg.test.mjs index 35a42028..8d225601 100644 --- a/tools/cli-package/gnupg.test.mjs +++ b/tools/cli-package/gnupg.test.mjs @@ -1,12 +1,13 @@ import assert from 'node:assert/strict'; -import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { mkdirSync, mkdtempSync, rmSync, statSync, symlinkSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; -import { delimiter, isAbsolute, join, relative } from 'node:path'; +import { delimiter, dirname, isAbsolute, join, relative } from 'node:path'; import { after, test } from 'node:test'; import { CustodyError } from './custody-error.mjs'; import { GNUPG_FLOOR, assertHomeState, + assertNoGlobalConfig, createGnupgHome, encryptArgs, gnupgEnvironment, @@ -26,6 +27,9 @@ const refuses = (code) => (error) => error instanceof CustodyError && error.code test('gpg resolves from absolute PATH entries only and is required, never skipped', async () => { const gpg = resolveGpg(); assert.ok(gpg.command.length > 0 && gpg.prefix.length === 0); + // The installation's own gpgconf sits beside the resolved gpg. + assert.equal(dirname(gpg.gpgconf.command), dirname(gpg.command)); + assert.match(gpg.gpgconf.command, /gpgconf(?:\.exe)?$/u); // The real binary meets the floor; this is the prerequisite local `pnpm run ci` and CI rely on. const home = createGnupgHome(scratch, 'version'); const version = await gnupgVersion(gpg, { @@ -125,7 +129,7 @@ test('every call is isolated, and encryption never passes a flag that widens it' for (const flag of required) assert.ok(args.includes(flag), flag); assert.equal(args[args.indexOf('--homedir') + 1], 'H'); for (const flag of forbidden) assert.ok(!args.includes(flag), flag); - assert.ok(!args.some((arg) => arg.startsWith('--allow-'))); + assert.ok(!args.some((arg) => arg.startsWith('--allow-') || arg.startsWith('--ignore-'))); } const encrypt = encryptArgs('H', 'K', 'I', 'O'); for (const [flag, value] of [ @@ -145,6 +149,36 @@ test('every call is isolated, and encryption never passes a flag that widens it' assert.equal(encrypt.at(-1), 'I'); }); +test('the environment is isolated in the C locale and every home is owner-only', () => { + const env = gnupgEnvironment(scratch); + assert.equal(env.LANG, 'C'); + assert.equal(env.LC_ALL, 'C'); + for (const name of ['HOME', 'TMPDIR', 'XDG_CONFIG_HOME']) assert.equal(env[name], scratch, name); + for (const name of ['GNUPGHOME', 'GPG_AGENT_INFO', 'GPG_TTY', 'GNUPG_HOME']) { + assert.equal(env[name], undefined, name); + } + const home = createGnupgHome(scratch, 'owner-only'); + if (process.platform !== 'win32') assert.equal(statSync(home).mode & 0o777, 0o700); +}); + +test('a global configuration in the sysconfdir gpgconf names refuses', async () => { + const gpg = { command: 'gpg', prefix: [], gpgconf: { command: 'gpgconf', prefix: [] } }; + const check = (stdout) => + assertNoGlobalConfig(gpg, { run: async () => ({ stdout, stderr: '' }), env: {} }); + const empty = createGnupgHome(scratch, 'sysconf-empty'); + await check(`${empty}\n`); + const planted = createGnupgHome(scratch, 'sysconf-planted'); + writeFileSync(join(planted, 'gpg.conf'), 'encrypt-to 0123456789ABCDEF\n'); + await assert.rejects(check(`${planted}\n`), refuses('GPG_CONFIG')); + // Only the configuration files GnuPG reads count; another daemon's file is not one. + const other = createGnupgHome(scratch, 'sysconf-other'); + writeFileSync(join(other, 'scdaemon.conf'), 'disable-ccid\n'); + await check(`${other}\n`); + // The real installation has none, so every encryption may proceed. + const real = resolveGpg(); + await assertNoGlobalConfig(real, { run: runCommand, env: gnupgEnvironment(scratch) }); +}); + test('status parsing admits only status lines and keeps unknown keywords for the caller', () => { assert.deepEqual(parseStatus('[GNUPG:] BEGIN_ENCRYPTION 2 9\n[GNUPG:] NEWSIG\n'), [ { keyword: 'BEGIN_ENCRYPTION', args: ['2', '9'] }, diff --git a/tools/cli-package/gpg-fixture.mjs b/tools/cli-package/gpg-fixture.mjs index ca8f7c9a..c2ccf31a 100644 --- a/tools/cli-package/gpg-fixture.mjs +++ b/tools/cli-package/gpg-fixture.mjs @@ -1,8 +1,9 @@ /** * Test-only GnuPG fixtures (Step 9b.1): the committed synthetic public keys and their pins, and an - * ephemeral decryption key generated per run in a short owner-only home. That home's agent is stopped - * and its socket directory removed when the test ends; no secret key is written anywhere else, and - * none is ever committed. + * ephemeral decryption key generated per run in a short owner-only home. Its public export and, for + * the secret-material refusal, its armored secret export go to a second owner-only temporary + * directory. Both directories are removed and the home's agent stopped when the test file ends; no + * secret key is ever committed. */ import assert from 'node:assert/strict'; import { spawnSync } from 'node:child_process'; diff --git a/tools/cli-package/ustar.test.mjs b/tools/cli-package/ustar.test.mjs index e529c8d9..be694e38 100644 --- a/tools/cli-package/ustar.test.mjs +++ b/tools/cli-package/ustar.test.mjs @@ -132,9 +132,9 @@ test('the archive structure refuses duplicates, aliases, truncation, padding, co ]) { assert.throws(() => read(archive), refuses(code), code); } - // The caller's path rule runs inside the header check, before the entry's bytes are bounded. + // The caller's path rule runs inside the header check, before its mode and size are judged. assert.throws( - () => read(Buffer.concat([ustarEntry('outside.txt', Buffer.alloc(0), 0o600), end])), + () => read(Buffer.concat([ustarEntry('outside.txt', Buffer.alloc(8192), 0o644), end])), refuses('PATH'), ); }); From ade62b7e9f93b57217d3a4e42ad4cb482c554043 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 14:46:06 +0300 Subject: [PATCH 73/97] fix(repo): address the second Step 9b.1 review round Round 2 at dea32ea6 reported one medium: round 1's removal before publishing left the outer CLEANUP override, which turns an earlier refusal into CLEANUP when its work directory cannot be removed, with no test that failed without it. A refusal whose work directory then cannot be removed is now tested to report CLEANUP. Lows corrected, none deferred: - gpgconf prints a named directory raw; only the unnamed listing is percent-escaped. The answer is taken as printed, and anything but one absolute path ended by one newline refuses GPG_CONFIG. - No private root qualifies on Windows, where modes say nothing about access, so encryptOnce refuses there before any GnuPG call. - Pins must be strings: a value that only converts to a fingerprint refuses PIN_FORMAT. - Each container record is held to its own reader's bound through one RECORD_BYTES map shared by the manifest readers, the sealer and the container. - The ephemeral key's home is no longer created before its path check. - New tests: KEY_CONSIDERED count, sec without ssb, every call's environment and bounds, the private root's group bits, link and owner, work and plaintext modes, the literal ciphertext name, exact container bounds, the read-side entry bound, the build's read-back self-check, empty and block-aligned USTAR entries, a planted ambient environment. - Contract, runbook, reference index and register wording. A 30-mutant sweep over the corrected guards left no survivor. Refs: ADR-0111 Co-Authored-By: Claude Opus 5.5 --- docs/reference/cli/README.md | 2 +- docs/reference/cli/packaging.md | 43 ++++-- docs/roadmap/deferred-tasks.md | 16 +- .../phase-2.5.5-hardening-and-remediation.md | 50 +++++- docs/runbooks/local-dev-setup.md | 2 +- tools/cli-package/README.md | 5 +- tools/cli-package/container.mjs | 12 +- tools/cli-package/container.test.mjs | 72 ++++++++- tools/cli-package/custody.mjs | 25 ++- tools/cli-package/custody.test.mjs | 145 ++++++++++++++++-- tools/cli-package/gnupg.mjs | 20 ++- tools/cli-package/gnupg.test.mjs | 42 ++++- tools/cli-package/gpg-fixture.mjs | 6 +- tools/cli-package/manifest.mjs | 15 +- tools/cli-package/seal.mjs | 6 +- tools/cli-package/ustar.test.mjs | 4 +- 16 files changed, 389 insertions(+), 76 deletions(-) diff --git a/docs/reference/cli/README.md b/docs/reference/cli/README.md index 6144bb9c..639dfe27 100644 --- a/docs/reference/cli/README.md +++ b/docs/reference/cli/README.md @@ -13,5 +13,5 @@ Part of [reference/](../README.md). | [commands.md](commands.md) | The full `relavium` command surface (run, chat, chat-resume/list/export, agent run, list, create, import, export, logs, gate, init, provider) plus interactive TUI vs CI JSON mode. | | [home.md](home.md) | The bare-invocation interactive **Home** — the TTY gate, the read-only management strip over `history.db`, the Home↔chat mode machine + keymap, bracketed paste, the min-terminal degrade, and the signal/exit-code lifecycle (2.5.B / ADR-0054). | | [chat-session.md](chat-session.md) | The `relavium chat` agent-session REPL — entry, agent/model selection, the multi-turn loop, streaming, tool availability, the `--json` session-event stream, and exit code 4. | -| [packaging.md](packaging.md) | Wave 3 finite runtime graph, installed-package consumers and Step 8 native activation accepted on `development`; the Step 9a.1 candidate build and finalizer are accepted on `development` and Step 9a.2a's private manifest and symbolication are accepted on `development` and Step 9a.2b's product map controls and `package_test` receipt are accepted on `development`; the remaining Step 9 custody and release gates remain staged. | +| [packaging.md](packaging.md) | Wave 3 finite runtime graph, installed-package consumers and Step 8 native activation accepted on `development`; the Step 9a.1 candidate build and finalizer are accepted on `development` and Step 9a.2a's private manifest and symbolication are accepted on `development` and Step 9a.2b's product map controls and `package_test` receipt are accepted on `development`; Step 9b.1's private container, isolated GnuPG runner and encrypt-once library are implemented on `development` and under review (synthetic recipients only); the remaining Step 9 custody and release gates remain staged. | | [agent-run-fixture.md](agent-run-fixture.md) | The `relavium agent run --fixture` cassette format — a `StreamChunk[][]` JSON recording of an agent run's LLM stream, for deterministic offline replay. | diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 04da0815..e317c554 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -530,30 +530,41 @@ recorded in the [Wave 3 register](../../roadmap/phases/phase-2.5.5-hardening-and sealed staging in its own layout: `finalized.json`, `manifest.json`, `metafile-esm.json` and `maps/.js.map`, sorted, with no host path. The staging must hold exactly those records and the manifest's maps as regular files of at most 16 MiB each, at most 259 entries and - 128 MiB in total (the total refused before the archive is assembled). Every map's digest must + 128 MiB in total (the total refused before the archive is assembled). Each record must also + be within the bound its own reader applies (`RECORD_BYTES`: `finalized.json` 1 MiB, + `manifest.json` and `metafile-esm.json` 4 MiB), so the container never carries a record + recovery would refuse. Every map's digest must be the manifest's, and the finalizer's record must name exactly the manifest's outputs and maps with each map's size. The manifest records no digest for the metafile, which must parse - as a JSON object. Reading a container back applies the same checks, and a decrypted container + as a JSON object. The written container is read back before it is returned, so a path the + dialect's 100-byte name field cannot hold refuses (`CONTAINER_TAR_NAME`). Reading a container + back applies the same checks, and a decrypted container extracted with `tar -x` is a staging the manifest reader consumes unchanged. - **The isolated runner.** `gnupg.mjs` resolves `gpg` from absolute PATH entries only and refuses one older than GnuPG 2.2.8; the installation — its binary, its `gpgconf` and its - system-wide configuration directory — is trusted. Every call runs in a fresh owner-only home - the tool creates, with `--no-options --no-autostart --no-keyring`, the C locale and an - isolated environment, bounded by `runCommand`. GnuPG still reads the installation's + system-wide configuration directory — is trusted. Every call runs in the C locale and an + isolated environment, bounded by `runCommand`; every `gpg` call also runs in a fresh + owner-only home the tool creates, with `--no-options --no-autostart --no-keyring`. GnuPG + still reads the installation's `/gpg.conf` under `--no-options`, and its `[force]` options override the command line, so before any key is handled the installation's `gpgconf --list-dirs sysconfdir` is - read and a `gpg.conf` or `common.conf` there refuses (`GPG_CONFIG`): a global configuration - could add or hide a recipient, redirect the output or log the session key. The recipient listing (`--show-keys`) uses its own home + read — the one call without a home — and a `gpg.conf` or `common.conf` there refuses + (`GPG_CONFIG`): a global configuration + could add or hide a recipient, redirect the output or log the session key. `gpgconf` prints a + named directory raw, ended by one newline; anything but exactly one absolute path refuses. The recipient listing (`--show-keys`) uses its own home and the default trust model, because GnuPG 2.4 cannot list an expired or revoked key under `--trust-model always` without a trust database; that home may gain only GnuPG's own `trustdb.gpg`. Encryption uses `--recipient-file`, `--trust-model always` (the pinned fingerprints and the colon pre-check are the assurance), an AES-256 preference, no compression and a fixed literal name, in a home that may gain only `random_seed`. - **The recipient pre-check.** The recipient file is read bounded and copied, so the bytes - checked are the bytes encrypted to, and pins are exactly two 40-digit uppercase fingerprints - (OpenPGP v4). An armored secret key refuses from its bytes; other secret material refuses - from the listing as secret, or as unreadable where GnuPG cannot list it. The colon listing - then refuses, + checked are the bytes encrypted to, and pins are exactly two strings, each a 40-digit + uppercase fingerprint (OpenPGP v4). An armored secret key refuses from its bytes; secret + material GnuPG lists refuses as secret, and a file GnuPG cannot list refuses as unreadable. + Secret material GnuPG neither lists nor reads, such as binary secret packets after the + armored block, is admitted and never reaches the ciphertext, the record or a log (a named + limit in [deferred-tasks.md](../../roadmap/deferred-tasks.md#wave-3-step-9b1--named-limits-2026-10-25)). + The colon listing then refuses, in this order: secret material, a missing pinned primary, more than one listed key, a missing pinned subkey, revocation, expiry at the current time, unusability (invalid or disabled, no encryption capability, a creation time in the future, or an encryption subkey other than @@ -564,7 +575,9 @@ recorded in the [Wave 3 register](../../roadmap/phases/phase-2.5.5-hardening-and - **Encrypt once.** `encryptOnce` refuses before any GnuPG call when the destination already holds `relavium--private-maps.tar.gpg`, when the destination and the owner-only private root overlap (compared by device and inode, so a second spelling of one directory - does not hide the overlap), or when either is missing or the root is not owner-only. Success requires exit 0, one + does not hide the overlap), or when either is missing or the root is not an owner-only + directory of this user. No root qualifies on Windows, where modes say nothing about access + and no ACL check exists. Success requires exit 0, exactly one `KEY_CONSIDERED` for the pinned primary, one `BEGIN_ENCRYPTION 2 9` (SEIPDv1 with MDC and AES-256; an AEAD profile refuses) before one `END_ENCRYPTION`, exactly the expected work files, a ciphertext within the container's size plus 64 KiB whose public-key packets name @@ -592,10 +605,12 @@ recorded in the [Wave 3 register](../../roadmap/phases/phase-2.5.5-hardening-and PGP private-key block. - **Where it runs.** The custody tests are part of `test:acceptance-tools`, so they run in local `pnpm run ci` and the four Ubuntu jobs inside `ci`'s needs, against the runner's GnuPG; they - fail, never skip, without GnuPG ≥ 2.2.8 (`docs/runbooks/local-dev-setup.md`). They do not run + fail, never skip, without GnuPG ≥ 2.2.8 or with a global configuration + ([local-dev-setup.md](../../runbooks/local-dev-setup.md)). They do not run on Windows: no Windows job runs `test:acceptance-tools`, the image's only `gpg` is Git for Windows' MSYS2 build, never exercised here, and the tests use POSIX process and permission - tools. The 2.2.8 floor is enforced by the version check without being exercised. + tools; `encryptOnce` itself refuses there. The 2.2.8 floor is enforced by the version check + without being exercised. ## Package-test and release modes diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index 8f8cd62c..bfd3e183 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -786,12 +786,16 @@ Recorded with the Step 9b.1 implementation ([checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-implementation-checkpoint--2026-10-25)). **Trigger for every entry:** the next change to the file it names, or a release producer on that platform. -- [ ] **The custody tests do not run on Windows.** *Low; a named limit.* No Windows job runs - `test:acceptance-tools`; the Windows image's only `gpg` is Git for Windows' MSYS2 build in `usr\bin`, - never exercised here, and the tests use POSIX process and permission tools (`ps`, owner-only modes), so - `docs/runbooks/local-dev-setup.md` sends Windows developers to WSL. **Re-check:** before a release - producer runs on Windows, run the custody tests on that runner and record which `gpg.exe` `resolveGpg` - selects and whether its Windows path handling passes. *(low · tools/cli-package/gnupg.mjs)* +- [ ] **The custody tests do not run on Windows, and `encryptOnce` refuses there.** *Low; a named limit.* + No Windows job runs `test:acceptance-tools`; the Windows image's only `gpg` is Git for Windows' MSYS2 + build in `usr\bin`, never exercised here, and the tests use POSIX process and permission tools (`ps`, + owner-only modes), so `docs/runbooks/local-dev-setup.md` sends Windows developers to WSL. Windows modes + say nothing about access and no ACL check exists, so since round 2 (SEC-9B1-R2-2) no private root + qualifies there and every encryption refuses `PRIVATE_ROOT` before any GnuPG call. **Re-check:** before + a release producer runs on Windows, add an ACL-based owner-only check for the private root and its work + directory, then run the custody tests on that runner and record which `gpg.exe` `resolveGpg` selects + and whether its Windows path handling and gpgconf's line ending pass. *(low · + tools/cli-package/custody.mjs, tools/cli-package/gnupg.mjs)* - [ ] **An interrupted custody test run can leave its ephemeral key home.** *Low; test-only.* *(rigor pass, SEC-9B1-PS-2)* `gpg-fixture.mjs` removes the round-trip key's home, its synthetic secret export and its agent in `after()`, which a SIGINT to `node --test` skips: the `rg-*` and `rge-*` directories under the diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index e2334b15..d0d06147 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -3702,8 +3702,8 @@ No credential values or request captures belong in tracked evidence. | 9a.2a | **Recorded split, 2026-10-24 (maintainer-approved) — decoder, private manifest and symbolication:** an in-house strict source-map decoder; strict mapping and range validation of upstream and final maps in the finalizer; the private map manifest (`staging/manifest.json`, keyed by `dist/.js`, with a per-source digest inventory and the commit's tree), sealed after the archive inspection; real packed CLI and `@relavium/core` frame capture through a tooling-only preload during an offline `relavium import`, run on the finalized `dist` that the inspection proves byte-equal to the tarball; symbolication of those frames against the matching source revision (clean: the checkout's Git objects; dirty, local only: the candidate copy), wired into the seal before the receipt is written. The receipt stays `compiled_package_test`. Security sitting. *Independently accepted on `development` at `a92a7602` on 2026-10-24 ([checkpoint](#wave-3-step-9a2a-implementation-checkpoint--2026-10-24), [acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)).* | | 9a.2b | **Recorded split, 2026-10-24 (maintainer-approved) — product map controls and the package-test receipt:** `check:map-controls` over product copies (wrong JS by line and column, wrong map, permuted sources, wrong source, missing, malformed and truncated upstream maps, a shifted valid upstream map, and the package-test producer's post-receipt rebuild refusals that ADR-0111 lists), run in `pnpm run ci` and as a required uncached CI job inside `ci`'s needs (the required contexts stay `ci` and `coverage`); then the atomic `package_test` flip across seal, binding, consumers, aggregate and both workflows, version 1 kept. Security sitting. *Preflight decisions approved on 2026-10-24 ([decisions](#wave-3-step-9a2b-preflight-decisions--2026-10-24)); independently accepted on `development` at `0f73c253` on 2026-10-24 ([checkpoint](#wave-3-step-9a2b-implementation-checkpoint--2026-10-24), [acceptance](#wave-3-step-9a2b-acceptance--2026-10-24)).* | | NI | **Native file identity on Windows (maintainer-approved 2026-10-24; after 9a.2b, before 9b):** every file and directory identity check compares `dev`/`ino` read with `{ bigint: true }` — the filesystem policy's pinned roots, the workflow catalog's open check and the media store's pins, read check and temporary claims — so a replaced file or directory can never compare equal on Windows, where `ino` is a 64-bit NTFS file ID that a JavaScript number cannot hold exactly; a replacement test per site. Found by 9a.2a's push CI ([acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)); recorded in [deferred-tasks.md](../deferred-tasks.md#native-file-identity-on-windows--approved-increment-2026-10-24). Security sitting. *Preflight decisions approved on 2026-10-25 ([decisions](#wave-3-ni-preflight-decisions--2026-10-25)); independently accepted on `development` at `fc5af701` on 2026-10-25 ([checkpoint](#wave-3-ni-implementation-checkpoint--2026-10-25), [acceptance](#wave-3-ni-acceptance--2026-10-25)); its Windows clause closes after three green advisory Windows runs on pushed heads.* | -| 9b | **Recorded split — recipient custody:** synthetic-only isolated GnuPG controls (public-only home/keyring, pinned primary and encryption-subkey fingerprints, machine-readable refusal of missing, expired, revoked or unusable recipients), encrypt once and reuse the original ciphertext. Explicit crypto/custody security sitting. Fixture proof never claims production recipient or custody configuration. | -| 9b.1 | **Recorded split, 2026-10-25 (maintainer-approved) — container, isolated GnuPG runner and encrypt-once:** the strict USTAR dialect extracted from the archive reader with its writer promoted; the private container built from the sealed staging in a staging-relative layout, each record's digest and size checked against the private manifest; an isolated public-only GnuPG runner (GnuPG ≥ 2.2.8; `--recipient-file` with no keyring, agent or trustdb); a colon-listing pre-check that refuses missing, expired, revoked, unusable (v4 with cv25519 or RSA ≥ 3072 only) or unexpected recipients and secret material; `encryptOnce` (SEIPDv1 with MDC and AES-256, exactly one `ENC_TO` for the pinned subkey, the exact produced asset, a no-clobber publish, plaintext cleanup on every path); committed synthetic public fixtures, an ephemeral round-trip key, and the GnuPG prerequisite documented. Library and synthetic controls only: product paths and workflows stay unchanged. Crypto/custody security sitting. *Implemented on `development` and under review ([decisions](#wave-3-step-9b-preflight-decisions--2026-10-25), [checkpoint](#wave-3-step-9b1-implementation-checkpoint--2026-10-25)).* | +| 9b | **Recorded split — recipient custody:** synthetic-only isolated GnuPG controls (public-only home/keyring, pinned primary and encryption-subkey fingerprints, machine-readable refusal of missing, expired, revoked or unusable recipients), encrypt once and reuse the original ciphertext. Explicit crypto/custody security sitting. Fixture proof never claims production recipient or custody configuration. *Split on 2026-10-25 (maintainer-approved, L8) into 9b.1 and 9b.2 below, which replace it as increments ([preflight decisions](#wave-3-step-9b-preflight-decisions--2026-10-25)).* | +| 9b.1 | **Recorded split, 2026-10-25 (maintainer-approved) — container, isolated GnuPG runner and encrypt-once:** the strict USTAR dialect extracted from the archive reader with its writer promoted; the private container built from the sealed staging in a staging-relative layout, each record's digest and size checked against the private manifest; an isolated public-only GnuPG runner (GnuPG ≥ 2.2.8; `--recipient-file` with no keyring or agent, and no trustdb but the empty one GnuPG may create in the listing home); a colon-listing pre-check that refuses missing, expired, revoked, unusable (v4 with cv25519 or RSA ≥ 3072 only) or unexpected recipients and secret material; `encryptOnce` (SEIPDv1 with MDC and AES-256, exactly one `ENC_TO` for the pinned subkey, the exact produced asset, a no-clobber publish, plaintext cleanup on every path); committed synthetic public fixtures, an ephemeral round-trip key, and the GnuPG prerequisite documented. Library and synthetic controls only: product paths and workflows stay unchanged. Crypto/custody security sitting. *Implemented on `development` and under review ([decisions](#wave-3-step-9b-preflight-decisions--2026-10-25), [checkpoint](#wave-3-step-9b1-implementation-checkpoint--2026-10-25)).* | | 9b.2 | **Recorded split, 2026-10-25 (maintainer-approved) — reuse, binding receipt and scope separation:** retained-ciphertext reuse without re-encryption (digest and size verified, zero GnuPG calls); the release binding receipt's schema v1, its validator and a sealer that writes only after the ciphertext exists (9d adds the origin, map-check and `ci`-run binding); the production recipient loader, absent in Wave 3; synthetic and production scope separation. Crypto/custody security sitting. | | 9c | **Recorded split — offline release state controller:** permission-enforcing offline GitHub/npm controller, permissioned prebuild discovery and reservation, and every rerun, partial, conflict and expiry state, with no overwrite, rebuild or re-encryption on uncertain history. | | 9d | **Recorded split — workflow, runbook and whole-Step-9 landing:** least-privilege actual workflow dependency graph, required artifact/platform aggregation and exact-artifact consumers, lost-staging refusal and the release runbook update. The whole-Step-9 cumulative review retains all 9a–9d changes with preservation traces into Step 8, and the dedicated archive/crypto security sitting closes the step. | @@ -10203,3 +10203,49 @@ Lows corrected: Two lows are recorded in deferred-tasks.md: secret material GnuPG neither lists nor reads is admitted, and the atomic publish has no fault-injection test. Every corrected guard's mutant now fails a test. +**Author evidence at `dea32ea6`.** The full `pnpm run ci` passed alone (exit 0): `test:acceptance-tools` 975 of +975. On Ubuntu 24.04 with GnuPG 2.4.4 (in Docker), the 41 USTAR, container, GnuPG and custody tests ran: 40 +passed and the case-insensitive-volume case skipped, as it does on a case-sensitive volume. + +**Round 2 at `dea32ea6` — NONCLEAN.** Five lenses and a critic ran, every finding adversarially verified. The plan +and integration lenses were clean. One medium, corrected: +- **The outer `CLEANUP` override was untested.** Round 1 added a removal before publishing, and the only `CLEANUP` + test then reached that inner removal alone. Deleting the outer override, which turns an earlier refusal into + `CLEANUP` when its work directory cannot be removed, failed no test, so a caller could be told `STATUS` while + plaintext remained. A refusal whose work directory then cannot be removed is now tested to report `CLEANUP`. + +Lows corrected, none deferred: +- **gpgconf's answer.** `gpgconf --list-dirs sysconfdir` prints a named directory raw; only the unnamed listing + is percent-escaped. The percent-decoding turned a literal `%41` in the path into another directory, so a global + configuration there was admitted. The answer is now taken as printed, and anything but exactly one absolute + path ended by one newline refuses (CRIT-9B1-R2-1; checked against the real gpgconf 2.5.21). +- **Windows.** The owner-only check admitted any directory on Windows, where modes say nothing about access. No + private root qualifies there now, so `encryptOnce` refuses before any GnuPG call, and the named limit says so + (SEC-9B1-R2-2). +- **Pins.** A pin that only converts to a fingerprint, such as a one-element array, passed. Pins must now be + strings (CRIT-9B1-R2-2). +- **Record bounds.** The container admitted a `manifest.json` of up to 16 MiB, which the manifest reader refuses + over 4 MiB. Each record is now held to its own reader's bound, one `RECORD_BYTES` map shared by the readers + and the container (CRIT-9B1-R2-3). +- **Fixture.** The ephemeral key's home was created before its path-length check, so a failed check left it + behind (T-9B1-R2-8). +- **Tests.** New cases cover: + - a duplicated or second `KEY_CONSIDERED`, and a `sec` record without `ssb`; + - the environment and bounds of every GnuPG call; + - the private root's group bits, a linked root and another user's root; + - the work directory's, homes' and plaintext files' modes; + - the ciphertext name as the contract spells it; + - every container bound met exactly, and the entry bound on reading; + - the build's read-back self-check, through a chunk name too long for the dialect's name field; + - empty and block-aligned USTAR entries; + - a planted ambient environment. + + (COR-9B1-R2-1/2, T-9B1-R2-2 to T-9B1-R2-7, SEC-9B1-R2-1.) +- **Wording.** The contract's secret-material sentence now names the recorded limit, its "every call" is + narrowed to every `gpg` call, and its runbook citation is a relative link. The runbook names the + global-configuration requirement. The 9b row carries its split note, and the CLI reference index names 9b.1 + (PLAN-9B1-R2-1/2/3, INT-9B1-R2-1/2, COR-9B1-R2-3). + +A 30-mutant sweep over the corrected guards and the round's named mutants left no survivor. That includes the +owner check's uid half, which no earlier test covered. + diff --git a/docs/runbooks/local-dev-setup.md b/docs/runbooks/local-dev-setup.md index 464839db..47a44164 100644 --- a/docs/runbooks/local-dev-setup.md +++ b/docs/runbooks/local-dev-setup.md @@ -21,7 +21,7 @@ For the full package layout this runbook references, see | Rust toolchain (`rustup`, `cargo`) | Builds the Tauri v2 desktop backend | Only needed to build `apps/desktop`. The engine, CLI, and tests do not need Rust. | | Tauri OS prerequisites | Native WebView + build deps | macOS: Xcode Command Line Tools. Windows: WebView2 + MSVC build tools. Linux: WebKitGTK + libsecret dev packages. See the [Tauri prerequisites guide](https://v2.tauri.app/start/prerequisites/). | | Git | Workflows and agents are git-native YAML | — | -| GnuPG ≥ 2.2.8 (`gpg`) | The release custody tests (`pnpm run ci`) encrypt synthetic map containers | Required, never skipped: the tests fail without it. macOS: `brew install gnupg`. Debian/Ubuntu: `apt install gnupg` (the CI Ubuntu runner has it). Windows: run these tests under WSL, since they use POSIX process and permission tools; the CI runs them on Ubuntu only. Release tooling, never a runtime dependency. | +| GnuPG ≥ 2.2.8 (`gpg`) | The release custody tests (`pnpm run ci`) encrypt synthetic map containers | Required, never skipped: the tests fail without it, and with a system-wide `gpg.conf` or `common.conf` in the directory `gpgconf --list-dirs sysconfdir` names (a global configuration refuses as `GPG_CONFIG`; Homebrew and Ubuntu ship that directory empty). macOS: `brew install gnupg`. Debian/Ubuntu: `apt install gnupg` (the CI Ubuntu runner has it). Windows: run these tests under WSL, since they use POSIX process and permission tools; the CI runs them on Ubuntu only. Release tooling, never a runtime dependency. | > You can do meaningful work on the engine and CLI **without** the Rust/Tauri > toolchain. Install Rust only when you start on `apps/desktop`. diff --git a/tools/cli-package/README.md b/tools/cli-package/README.md index 64f2f5e3..72f48f10 100644 --- a/tools/cli-package/README.md +++ b/tools/cli-package/README.md @@ -80,8 +80,9 @@ unless every expected control ran once with its expected outcome; the required ` runs it uncached and uploads nothing. Step 9b.1's custody library (synthetic recipients only; nothing calls it yet) builds one owner-only USTAR of the sealed staging (`container.mjs`, over the one strict dialect in `ustar.mjs`) and encrypts it once to one pinned recipient with GnuPG ≥ 2.2.8 in fresh -isolated homes (`custody.mjs` over `gnupg.mjs`): a colon pre-check refuses missing, expired, revoked, -unusable or unexpected recipients and secret material, the ciphertext must be SEIPDv1 with MDC and +isolated homes (`custody.mjs` over `gnupg.mjs`; POSIX only, since no private root qualifies on +Windows): a colon pre-check refuses missing, expired, revoked, unusable or unexpected recipients and the +secret material it can detect, the ciphertext must be SEIPDv1 with MDC and AES-256 naming exactly the pinned subkey, an existing ciphertext is never overwritten, the plaintext work directory is removed on every path and every refusal is a fixed `PACKED_CLI_CUSTODY_*` code. `custody-fixtures/` holds the committed synthetic public keys; no secret key is committed. No private map, diff --git a/tools/cli-package/container.mjs b/tools/cli-package/container.mjs index 463c6ead..f17f57da 100644 --- a/tools/cli-package/container.mjs +++ b/tools/cli-package/container.mjs @@ -12,7 +12,7 @@ import { join, posix } from 'node:path'; import { isDeepStrictEqual } from 'node:util'; import { CustodyError, refuseCustody } from './custody-error.mjs'; import { readOrdinaryFile, sha256 } from './files.mjs'; -import { MANIFEST_FILE, validateManifest } from './manifest.mjs'; +import { MANIFEST_FILE, RECORD_BYTES, validateManifest } from './manifest.mjs'; import { readUstar, writeUstar } from './ustar.mjs'; export const CONTAINER_LIMITS = Object.freeze({ @@ -62,12 +62,18 @@ function recorded(list) { } /** - * Check a container's entries as a whole: exactly the three records and one map per manifest output, - * each map the manifest's and the finalizer record's, the record naming exactly the manifest's files. + * Check a container's entries as a whole: exactly the three records, each within the bound its reader + * applies, and one map per manifest output, each map the manifest's and the finalizer record's, the + * record naming exactly the manifest's files. * @param {Map} files */ function checkContents(files) { for (const name of RECORDS) if (!files.has(name)) refuseCustody('CONTAINER_SET'); + for (const name of RECORDS) { + if (/** @type {Buffer} */ (files.get(name)).length > RECORD_BYTES[name]) { + refuseCustody('CONTAINER_RECORD'); + } + } const manifest = (() => { try { return validateManifest( diff --git a/tools/cli-package/container.test.mjs b/tools/cli-package/container.test.mjs index 88d60c0f..710bd441 100644 --- a/tools/cli-package/container.test.mjs +++ b/tools/cli-package/container.test.mjs @@ -6,16 +6,17 @@ import { mkdirSync, readFileSync, readdirSync, + renameSync, rmSync, symlinkSync, writeFileSync, } from 'node:fs'; -import { join } from 'node:path'; +import { join, posix } from 'node:path'; import { before, test } from 'node:test'; import { candidate, fresh, posixModes, seal } from './candidate-fixture.mjs'; import { CONTAINER_LIMITS, buildPrivateContainer, readPrivateContainer } from './container.mjs'; import { CustodyError } from './custody-error.mjs'; -import { readPrivateManifest } from './manifest.mjs'; +import { RECORD_BYTES, readPrivateManifest } from './manifest.mjs'; import { writeUstar } from './ustar.mjs'; const refuses = (code) => (error) => error instanceof CustodyError && error.code === code; @@ -252,6 +253,19 @@ test('the entry, count and total bounds refuse before a container is written', ( refuses('CONTAINER_SIZE'), ); const { bytes } = buildPrivateContainer(copy); + // Exactly at every bound is admitted, on building and on reading. + const exact = { + entryBytes: largest, + entries: 3 + readdirSync(join(copy, 'maps')).length, + totalBytes: bytes.length, + }; + assert.deepEqual(buildPrivateContainer(copy, exact).bytes, bytes); + assert.equal(readPrivateContainer(bytes, exact).files.size, exact.entries); + // Reading bounds each entry as building does. + assert.throws( + () => readPrivateContainer(bytes, { ...CONTAINER_LIMITS, entryBytes: largest - 1 }), + refuses('CONTAINER_FILE_SIZE'), + ); // The total is refused before the archive is assembled, not by the read-back check. assert.throws( () => buildPrivateContainer(copy, { ...CONTAINER_LIMITS, totalBytes: bytes.length - 1 }), @@ -267,6 +281,60 @@ test('the entry, count and total bounds refuse before a container is written', ( ); }); +test("each record stays within its reader's bound, on building and on reading", () => { + /** @param {string} name @param {number} extra */ + const padded = (name, extra) => + staging((copy) => { + const path = join(copy, name); + const bytes = readFileSync(path); + // Trailing whitespace keeps the JSON valid, so only the size is judged. + const padding = Buffer.alloc(RECORD_BYTES[name] - bytes.length + extra, 0x20); + writeFileSync(path, Buffer.concat([bytes, padding])); + }); + for (const name of ['finalized.json', 'manifest.json', 'metafile-esm.json']) { + buildPrivateContainer(padded(name, 0)); + assert.throws(() => buildPrivateContainer(padded(name, 1)), refuses('CONTAINER_RECORD'), name); + } + // The manifest reader recovery uses refuses that manifest too. + assert.throws( + () => readPrivateManifest(padded('manifest.json', 1)), + (error) => error instanceof Error && 'code' in error && error.code === 'UNREADABLE', + ); + const files = readPrivateContainer(buildPrivateContainer(staging()).bytes).files; + const over = new Map(files); + over.set( + 'manifest.json', + Buffer.concat([files.get('manifest.json'), Buffer.alloc(RECORD_BYTES['manifest.json'], 0x20)]), + ); + assert.throws( + () => readPrivateContainer(writeUstar([...over], () => 0o600)), + refuses('CONTAINER_RECORD'), + ); +}); + +test('a written container that does not read back refuses: the self-check', () => { + // A chunk name too long for the dialect's 100-byte name field is truncated by the writer, and + // only the read-back of the written container notices. + const copy = staging((dir) => { + const manifest = JSON.parse(readFileSync(join(dir, 'manifest.json'), 'utf8')); + const record = JSON.parse(readFileSync(join(dir, 'finalized.json'), 'utf8')); + const [output] = manifest.outputs; + const long = `dist/${'x'.repeat(100)}.js`; + renameSync( + join(dir, 'maps', posix.basename(output.map.path)), + join(dir, 'maps', `${posix.basename(long)}.map`), + ); + for (const item of record.outputs) if (item.path === output.path) item.path = long; + for (const item of record.maps) if (item.path === output.map.path) item.path = `${long}.map`; + output.path = long; + output.map.path = `${long}.map`; + manifest.outputs.sort((left, right) => (left.path < right.path ? -1 : 1)); + writeFileSync(join(dir, 'manifest.json'), JSON.stringify(manifest)); + writeFileSync(join(dir, 'finalized.json'), JSON.stringify(record)); + }); + assert.throws(() => buildPrivateContainer(copy), refuses('CONTAINER_TAR_NAME')); +}); + test('a container outside the owner-only dialect, or whose contents disagree, refuses on reading', () => { const files = readPrivateContainer(buildPrivateContainer(staging()).bytes).files; const write = (entries, mode = 0o600) => writeUstar(entries, () => mode); diff --git a/tools/cli-package/custody.mjs b/tools/cli-package/custody.mjs index 6034494b..fbe383a7 100644 --- a/tools/cli-package/custody.mjs +++ b/tools/cli-package/custody.mjs @@ -71,16 +71,17 @@ export const ciphertextName = (version) => `relavium-${version}-private-maps.tar * @returns {{primary: string, subkey: string}} */ export function parsePins(pins) { + if (pins === null || typeof pins !== 'object') return refuseCustody('PIN_FORMAT'); + const { primary, subkey } = /** @type {Record} */ (pins); if ( - pins === null || - typeof pins !== 'object' || !isDeepStrictEqual(Object.keys(pins).sort(), ['primary', 'subkey']) || - !FINGERPRINT.test(String(/** @type {Record} */ (pins).primary)) || - !FINGERPRINT.test(String(/** @type {Record} */ (pins).subkey)) + typeof primary !== 'string' || + typeof subkey !== 'string' || + !FINGERPRINT.test(primary) || + !FINGERPRINT.test(subkey) ) { refuseCustody('PIN_FORMAT'); } - const { primary, subkey } = /** @type {{primary: string, subkey: string}} */ (pins); return Object.freeze({ primary, subkey }); } @@ -179,21 +180,19 @@ function plainDirectory(path) { } } -/** An owner-only directory of this user, not a link; a missing path is not one. */ +/** + * An owner-only directory of this user, not a link; a missing path is not one. Windows modes say + * nothing about access and no ACL check exists, so no directory there is one. + */ function privateDirectory(path) { + if (process.platform === 'win32') return false; let info; try { info = lstatSync(path); } catch { return false; } - return ( - info.isDirectory() && - !info.isSymbolicLink() && - (process.platform === 'win32' || - ((info.mode & 0o077) === 0 && - (typeof process.getuid !== 'function' || info.uid === process.getuid()))) - ); + return info.isDirectory() && (info.mode & 0o077) === 0 && info.uid === process.getuid?.(); } /** diff --git a/tools/cli-package/custody.test.mjs b/tools/cli-package/custody.test.mjs index 94a4a371..1de4430d 100644 --- a/tools/cli-package/custody.test.mjs +++ b/tools/cli-package/custody.test.mjs @@ -10,10 +10,11 @@ import { readFileSync, readdirSync, rmSync, + statSync, symlinkSync, writeFileSync, } from 'node:fs'; -import { dirname, join } from 'node:path'; +import { basename, dirname, join } from 'node:path'; import { before, test } from 'node:test'; import { fileURLToPath } from 'node:url'; import { candidate, fresh, posixModes, seal } from './candidate-fixture.mjs'; @@ -21,6 +22,7 @@ import { buildPrivateContainer } from './container.mjs'; import { CustodyError } from './custody-error.mjs'; import { ciphertextName, encryptOnce, inspectRecipient, parsePins } from './custody.mjs'; import { + GNUPG_CALL, createGnupgHome, gnupgEnvironment, gnupgVersion, @@ -55,7 +57,8 @@ before(async () => { staging = fixtureCandidate.staging; container = buildPrivateContainer(staging).bytes; }); -const NAME = () => ciphertextName(readPrivateManifest(staging).packageVersion); +/** The contract's ciphertext name, spelled independently of `ciphertextName`. */ +const NAME = () => `relavium-${readPrivateManifest(staging).packageVersion}-private-maps.tar.gpg`; /** A fresh owner-only private root and an empty destination. */ function places() { @@ -94,6 +97,11 @@ function recording(hooks = {}) { const run = async (command, args, options) => { const kind = kindOf(args); calls.push(kind); + // Every call runs in the work directory's isolated C-locale environment, within the call bounds. + assert.deepEqual(options.env, gnupgEnvironment(options.env.HOME)); + assert.match(basename(options.env.HOME), /^custody-/u); + assert.equal(options.timeoutMs, GNUPG_CALL.timeoutMs); + assert.equal(options.maxBytes, GNUPG_CALL.maxBytes); const hook = hooks[kind] ?? {}; if (hook.before) await hook.before(args); if (hook.replace) return hook.replace(args); @@ -169,6 +177,10 @@ test('pins are exactly two 40-digit uppercase fingerprints', () => { { primary: PRIMARY, subkey: SUBKEY.slice(-16) }, { primary: 'good ', subkey: SUBKEY }, { primary: 'C'.repeat(64), subkey: SUBKEY }, + // A value that only converts to a fingerprint is not one. + { primary: [PRIMARY], subkey: [SUBKEY] }, + { primary: PRIMARY, subkey: [SUBKEY] }, + { primary: { toString: () => PRIMARY }, subkey: SUBKEY }, ]) { assert.throws(() => parsePins(pins), refuses('PIN_FORMAT'), JSON.stringify(pins)); } @@ -179,6 +191,7 @@ test('the recipient classification refuses in its fixed order', () => { assert.equal(inspectRecipient(records(), pins, NOW), pins); for (const [code, listing, override] of [ ['RECIPIENT_SECRET', [key('sec'), fpr(PRIMARY), key('ssb'), fpr(SUBKEY)]], + ['RECIPIENT_SECRET', [key('sec'), fpr(PRIMARY), key('sub'), fpr(SUBKEY)]], ['RECIPIENT_MISSING', []], ['RECIPIENT_MISSING', records(), { primary: 'C'.repeat(40), subkey: SUBKEY }], ['RECIPIENT_MISSING', records(), { primary: PRIMARY, subkey: 'C'.repeat(40) }], @@ -318,6 +331,7 @@ test('encryption publishes exactly one ciphertext for the pinned subkey and leav assert.ok(Object.isFrozen(outcome.record) && Object.isFrozen(outcome.record.recipient)); assert.deepEqual(calls, ['version', 'config', 'list', 'encrypt', 'recipients']); assert.deepEqual(readdirSync(outcome.root), []); + assert.equal(ciphertextName('1.2.3'), 'relavium-1.2.3-private-maps.tar.gpg'); }); test('an ephemeral key decrypts the ciphertext to the container, and tar extracts the staging', async () => { @@ -407,6 +421,8 @@ test('the status must be exactly one pinned, completed SEIPDv1 AES-256 encryptio ), ['STATUS', (text) => `${text}[GNUPG:] END_ENCRYPTION\n`], ['STATUS', (text) => text.replace(/\[GNUPG:\] KEY_CONSIDERED[^\n]*\n/u, '')], + ['STATUS', (text) => text.replace(/(\[GNUPG:\] KEY_CONSIDERED[^\n]*\n)/u, '$1$1')], + ['STATUS', (text) => `${text}[GNUPG:] KEY_CONSIDERED ${'A'.repeat(40)} 0\n`], [ 'STATUS', (text) => @@ -564,13 +580,12 @@ test("a global GnuPG configuration in the installation's sysconfdir refuses befo refused(await attempt({ run }), 'GPG_CONFIG'); assert.deepEqual(calls, ['version', 'config'], name); } - // gpgconf percent-escapes a directory name; a relative or malformed answer refuses. - const escaped = fresh('sys conf'); - mkdirSync(escaped); - writeFileSync(join(escaped, 'gpg.conf'), 'throw-keyids\n'); - refused(await attempt(recording(config(`${escaped.replaceAll(' ', '%20')}\n`))), 'GPG_CONFIG'); + // gpgconf prints a named directory raw: a space or a literal `%41` is part of the path. + const raw = fresh('sys conf%41'); + mkdirSync(raw); + writeFileSync(join(raw, 'gpg.conf'), 'throw-keyids\n'); + refused(await attempt(recording(config(`${raw}\n`))), 'GPG_CONFIG'); refused(await attempt(recording(config('relative/dir\n'))), 'GPG_CONFIG'); - refused(await attempt(recording(config('%zz\n'))), 'GPG_CONFIG'); refused( await attempt( recording({ @@ -681,11 +696,24 @@ test('the destination and the private root are separate directories, the root ow encryptOnce(options({ destination: linked, privateRoot: root })), refuses('PLACEMENT'), ); - const open = fresh('open-root'); - mkdirSync(open); - chmodSync(open, 0o755); + for (const mode of [0o755, 0o750, 0o770]) { + const open = fresh('open-root'); + mkdirSync(open); + chmodSync(open, mode); + await assert.rejects( + encryptOnce(options({ destination, privateRoot: open })), + refuses('PRIVATE_ROOT'), + mode.toString(8), + ); + } + // A link to an owner-only directory is not one. + const target = fresh('linked-root-target'); + mkdirSync(target); + chmodSync(target, 0o700); + const linkedRoot = fresh('linked-root'); + symlinkSync(target, linkedRoot); await assert.rejects( - encryptOnce(options({ destination, privateRoot: open })), + encryptOnce(options({ destination, privateRoot: linkedRoot })), refuses('PRIVATE_ROOT'), ); } @@ -698,6 +726,69 @@ test('the destination and the private root are separate directories, the root ow assert.deepEqual(calls, []); }); +test('a private root of another user, or any on Windows, refuses before any GnuPG call', async () => { + const { root, destination } = places(); + const { run, calls } = recording(); + const attemptHere = () => + encryptOnce({ + container, + recipient: { file: fixture('good'), pins: PINS.good }, + destination, + privateRoot: root, + run, + }); + // Windows modes say nothing about access, and no ACL check exists. + const platform = Object.getOwnPropertyDescriptor(process, 'platform'); + Object.defineProperty(process, 'platform', { ...platform, value: 'win32' }); + try { + await assert.rejects(attemptHere(), refuses('PRIVATE_ROOT')); + } finally { + Object.defineProperty(process, 'platform', platform); + } + if (posixModes) { + // An owner-only directory that belongs to someone else is not this user's. + const getuid = process.getuid; + process.getuid = () => getuid.call(process) + 1; + try { + await assert.rejects(attemptHere(), refuses('PRIVATE_ROOT')); + } finally { + process.getuid = getuid; + } + } + assert.deepEqual(calls, []); +}); + +test( + 'the work directory, its homes and its plaintext files are owner-only', + { skip: !posixModes }, + async () => { + /** @type {Record} */ + const modes = {}; + const mode = (path) => statSync(path).mode & 0o777; + const { run } = recording({ + encrypt: { + before: (args) => { + const work = dirname(valueOf(args, '--output')); + modes.work = mode(work); + modes.homes = ['listing', 'encryption', 'recipients'].map((name) => + mode(join(work, name)), + ); + modes.container = mode(args.at(-1)); + modes.recipient = mode(valueOf(args, '--recipient-file')); + }, + }, + }); + const outcome = await attempt({ run }); + assert.equal(outcome.error, undefined, String(outcome.error)); + assert.deepEqual(modes, { + work: 0o700, + homes: [0o700, 0o700, 0o700], + container: 0o600, + recipient: 0o600, + }); + }, +); + test('an overlap spelled differently on a case-insensitive volume still refuses', async (context) => { const base = fresh('case'); mkdirSync(join(base, 'Public'), { recursive: true }); @@ -768,6 +859,36 @@ test( } finally { chmodSync(root, 0o700); } + // A refusal whose work directory then cannot be removed reports CLEANUP, never its own code. + const failed = places(); + const refusing = recording({ + encrypt: { + after: (result) => { + chmodSync(failed.root, 0o500); + return { ...result, stdout: `${result.stdout}[GNUPG:] FAILURE encrypt 53\n` }; + }, + }, + }); + try { + await assert.rejects( + encryptOnce({ + container, + recipient: { file: fixture('good'), pins: PINS.good }, + destination: failed.destination, + privateRoot: failed.root, + run: refusing.run, + }), + (error) => + error instanceof CustodyError && + error.code === 'CLEANUP' && + !JSON.stringify({ message: error.message, ...error }).includes(failed.root), + ); + assert.deepEqual(refusing.calls, ['version', 'config', 'list', 'encrypt']); + assert.equal(readdirSync(failed.root).length, 1, 'the work directory was removed'); + assert.deepEqual(readdirSync(failed.destination), []); + } finally { + chmodSync(failed.root, 0o700); + } }, ); diff --git a/tools/cli-package/gnupg.mjs b/tools/cli-package/gnupg.mjs index 757ae5fa..8f01ae7e 100644 --- a/tools/cli-package/gnupg.mjs +++ b/tools/cli-package/gnupg.mjs @@ -1,8 +1,9 @@ /** * The isolated GnuPG boundary of the custody tools (Step 9b.1; packaging.md, "Recipient custody"). - * GnuPG is release tooling, never a runtime dependency. Every call runs the resolved absolute binary - * through the shell-free, bounded `runCommand`, in a fresh owner-only home the tools create themselves, - * with no per-home options file, keyring, agent or key discovery. GnuPG still reads its installation's + * GnuPG is release tooling, never a runtime dependency. Every call runs a resolved absolute binary + * through the shell-free, bounded `runCommand` in the isolated environment. Every `gpg` call runs in a + * fresh owner-only home the tools create themselves, with no per-home options file, keyring, agent or + * key discovery; the one `gpgconf` query takes no home. GnuPG still reads its installation's * system-wide `gpg.conf` under `--no-options`, so a global configuration refuses before any call that * handles a key. Only status lines, colon records, the version line and the configuration directory * are read; other human-readable output and stderr are never read or surfaced. @@ -211,14 +212,11 @@ export async function assertNoGlobalConfig(gpg, { run, env }) { } catch { return refuseCustody('GPG_UNAVAILABLE'); } - let directory; - try { - // gpgconf percent-escapes special characters in the directories it lists. - directory = decodeURIComponent(String(stdout).trim()); - } catch { - return refuseCustody('GPG_CONFIG'); - } - if (!isAbsolute(directory)) refuseCustody('GPG_CONFIG'); + // A named directory is printed raw and ended by one newline; only the unnamed listing is + // percent-escaped. Anything that is not exactly one absolute path refuses. + const text = String(stdout); + const directory = text.endsWith('\n') ? text.slice(0, -1) : ''; + if (!isAbsolute(directory) || /[\r\n\0]/u.test(directory)) refuseCustody('GPG_CONFIG'); for (const name of ['gpg.conf', 'common.conf']) { if (existsSync(join(directory, name))) refuseCustody('GPG_CONFIG'); } diff --git a/tools/cli-package/gnupg.test.mjs b/tools/cli-package/gnupg.test.mjs index 8d225601..3b49366e 100644 --- a/tools/cli-package/gnupg.test.mjs +++ b/tools/cli-package/gnupg.test.mjs @@ -150,13 +150,35 @@ test('every call is isolated, and encryption never passes a flag that widens it' }); test('the environment is isolated in the C locale and every home is owner-only', () => { - const env = gnupgEnvironment(scratch); + // Planted in this process's environment, so a builder that leaked it would carry them. + const planted = { + GNUPGHOME: '/planted', + GPG_AGENT_INFO: '/planted', + GPG_TTY: '/planted', + GNUPG_HOME: '/planted', + LD_PRELOAD: '/planted', + DYLD_INSERT_LIBRARIES: '/planted', + LANG: 'tr_TR.UTF-8', + LC_ALL: 'tr_TR.UTF-8', + }; + const saved = Object.keys(planted).map((name) => [name, process.env[name]]); + Object.assign(process.env, planted); + let env; + try { + env = gnupgEnvironment(scratch); + } finally { + for (const [name, value] of saved) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + } assert.equal(env.LANG, 'C'); assert.equal(env.LC_ALL, 'C'); for (const name of ['HOME', 'TMPDIR', 'XDG_CONFIG_HOME']) assert.equal(env[name], scratch, name); - for (const name of ['GNUPGHOME', 'GPG_AGENT_INFO', 'GPG_TTY', 'GNUPG_HOME']) { + for (const name of Object.keys(planted).filter((key) => !['LANG', 'LC_ALL'].includes(key))) { assert.equal(env[name], undefined, name); } + assert.ok(!Object.values(env).includes('/planted')); const home = createGnupgHome(scratch, 'owner-only'); if (process.platform !== 'win32') assert.equal(statSync(home).mode & 0o777, 0o700); }); @@ -174,6 +196,22 @@ test('a global configuration in the sysconfdir gpgconf names refuses', async () const other = createGnupgHome(scratch, 'sysconf-other'); writeFileSync(join(other, 'scdaemon.conf'), 'disable-ccid\n'); await check(`${other}\n`); + // A named directory is printed raw: a literal `%41` is part of the path, never an escape. + const raw = createGnupgHome(scratch, 'sysconf %41'); + writeFileSync(join(raw, 'gpg.conf'), 'encrypt-to 0123456789ABCDEF\n'); + await assert.rejects(check(`${raw}\n`), refuses('GPG_CONFIG')); + // Anything but exactly one absolute path ended by one newline refuses. + for (const answer of [ + 'relative/dir\n', + '\n', + '', + empty, + `${empty}\n${empty}\n`, + `${empty}\r\n`, + `${empty}\0\n`, + ]) { + await assert.rejects(check(answer), refuses('GPG_CONFIG'), JSON.stringify(answer)); + } // The real installation has none, so every encryption may proceed. const real = resolveGpg(); await assertNoGlobalConfig(real, { run: runCommand, env: gnupgEnvironment(scratch) }); diff --git a/tools/cli-package/gpg-fixture.mjs b/tools/cli-package/gpg-fixture.mjs index c2ccf31a..12b4c18f 100644 --- a/tools/cli-package/gpg-fixture.mjs +++ b/tools/cli-package/gpg-fixture.mjs @@ -41,8 +41,10 @@ export function processesNaming(home) { */ export function ephemeralKey() { const gpg = resolveGpg(); - const home = mkdtempSync(join(tmpdir(), 'rg-')); - assert.ok(home.length + SOCKET.length < 100, 'the ephemeral key home path is too long'); + // Checked before anything is created: `mkdtemp` appends six characters to the prefix. + const prefix = join(tmpdir(), 'rg-'); + assert.ok(prefix.length + 6 + SOCKET.length < 100, 'the ephemeral key home path is too long'); + const home = mkdtempSync(prefix); const exportDirectory = mkdtempSync(join(tmpdir(), 'rge-')); const env = { PATH: process.env.PATH ?? '', LANG: 'C', LC_ALL: 'C' }; /** @param {string[]} args @param {string} [input] */ diff --git a/tools/cli-package/manifest.mjs b/tools/cli-package/manifest.mjs index eb020cca..0edc0143 100644 --- a/tools/cli-package/manifest.mjs +++ b/tools/cli-package/manifest.mjs @@ -50,6 +50,15 @@ const SOURCE = new RegExp( ); const VERSION = /^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]{1,64})?$/u; const LIMITS = Object.freeze({ outputs: 256, sources: 8192, recordBytes: 1024 * 1024 }); +/** + * The bound each staged record's reader applies, by file name: the custody container carries no record + * its reader would refuse. + */ +export const RECORD_BYTES = Object.freeze({ + 'finalized.json': LIMITS.recordBytes, + [MANIFEST_FILE]: LIMITS.recordBytes * 4, + 'metafile-esm.json': 4 * 1024 * 1024, +}); export class ManifestError extends Error { /** @param {string} code */ @@ -188,7 +197,9 @@ export async function sealPrivateManifest({ let record; try { record = JSON.parse( - readOrdinaryFile(join(staging, 'finalized.json'), LIMITS.recordBytes).toString('utf8'), + readOrdinaryFile(join(staging, 'finalized.json'), RECORD_BYTES['finalized.json']).toString( + 'utf8', + ), ); } catch { refuse('RECORD'); @@ -329,7 +340,7 @@ export function readPrivateManifest(staging) { let value; try { value = JSON.parse( - readOrdinaryFile(join(staging, MANIFEST_FILE), LIMITS.recordBytes * 4).toString('utf8'), + readOrdinaryFile(join(staging, MANIFEST_FILE), RECORD_BYTES[MANIFEST_FILE]).toString('utf8'), ); } catch { refuse('UNREADABLE'); diff --git a/tools/cli-package/seal.mjs b/tools/cli-package/seal.mjs index 9fcc2262..ebac284c 100644 --- a/tools/cli-package/seal.mjs +++ b/tools/cli-package/seal.mjs @@ -11,7 +11,7 @@ import { basename, isAbsolute, join } from 'node:path'; import { inspectPackageArchive } from './archive.mjs'; import { PACKAGE_TEST_MODE, originFromEnvironment, validatePackageReceipt } from './binding.mjs'; import { flatFiles, oneTarball, readOrdinaryFile, sha256 } from './files.mjs'; -import { MANIFEST_FILE, sealPrivateManifest, verifySealedDist } from './manifest.mjs'; +import { MANIFEST_FILE, RECORD_BYTES, sealPrivateManifest, verifySealedDist } from './manifest.mjs'; import { symbolicateCandidate } from './symbolicate.mjs'; export function producerNodeMajor(root) { @@ -59,7 +59,9 @@ export function expectedPackManifest(root) { export function finalizedPublicFiles(root, staging) { const cli = join(root, 'apps/cli'); const files = new Map(); - const graph = JSON.parse(readOrdinaryFile(join(staging, 'metafile-esm.json'), 4 * 1024 * 1024)); + const graph = JSON.parse( + readOrdinaryFile(join(staging, 'metafile-esm.json'), RECORD_BYTES['metafile-esm.json']), + ); const outputs = Object.keys(graph.outputs) .map((path) => path.replaceAll('\\', '/')) .filter((path) => !path.endsWith('.map')); diff --git a/tools/cli-package/ustar.test.mjs b/tools/cli-package/ustar.test.mjs index be694e38..a87b269c 100644 --- a/tools/cli-package/ustar.test.mjs +++ b/tools/cli-package/ustar.test.mjs @@ -42,9 +42,11 @@ function read(tar, overrides = {}) { return { files, order }; } +/** Unaligned, empty, exactly one block, and one byte over a block: every padding case. */ const ENTRIES = [ ['manifest.json', Buffer.from('{"synthetic":true}\n')], - ['maps/index.js.map', Buffer.from('{"version":3}')], + ['empty.json', Buffer.alloc(0)], + ['maps/index.js.map', Buffer.alloc(512, 0x62)], ['metafile-esm.json', Buffer.alloc(513, 0x61)], ]; From b3eaebaf96a5ee3d940201661a04bd46a01b0cd8 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 16:02:36 +0300 Subject: [PATCH 74/97] docs(repo): record the Step 9b.1 round 2 evidence The full pnpm run ci passed alone at ade62b7e (test:acceptance-tools 979 of 979, all 25 product map controls, every causal-control family); a first run that set the GitHub variables for the whole chain stopped at the map-controls origin gate by design and is recorded as such. The custody tests passed on Ubuntu 24.04 with GnuPG 2.4.4. The contract now says a container record stays within its own reader's bound. No code changes. Refs: ADR-0111 Co-Authored-By: Claude Opus 5.5 --- docs/reference/cli/packaging.md | 2 +- .../phases/phase-2.5.5-hardening-and-remediation.md | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index e317c554..15f4aadf 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -533,7 +533,7 @@ recorded in the [Wave 3 register](../../roadmap/phases/phase-2.5.5-hardening-and 128 MiB in total (the total refused before the archive is assembled). Each record must also be within the bound its own reader applies (`RECORD_BYTES`: `finalized.json` 1 MiB, `manifest.json` and `metafile-esm.json` 4 MiB), so the container never carries a record - recovery would refuse. Every map's digest must + its own reader would refuse. Every map's digest must be the manifest's, and the finalizer's record must name exactly the manifest's outputs and maps with each map's size. The manifest records no digest for the metafile, which must parse as a JSON object. The written container is read back before it is returned, so a path the diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index d0d06147..79ef5079 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -10249,3 +10249,11 @@ Lows corrected, none deferred: A 30-mutant sweep over the corrected guards and the round's named mutants left no survivor. That includes the owner check's uid half, which no earlier test covered. +**Author evidence at `ade62b7e`.** A first full run was started with `CI=true GITHUB_ACTIONS=true` set for the +whole chain. `check:map-controls` refuses such a half-configured GitHub environment before it builds anything +(`PACKED_CLI_MAP_CONTROL_ORIGIN`), as designed, so that run certified nothing. Rerun plain and alone, the full +`pnpm run ci` passed (exit 0): `test:acceptance-tools` 979 of 979, all 25 product map controls, and every +causal-control family. On Ubuntu 24.04 with GnuPG 2.4.4 (in Docker), the 45 USTAR, container, GnuPG and custody +tests ran: 44 passed and the case-insensitive-volume case skipped. The commit recording this evidence also +corrects one contract phrase and changes no code. + From 147c2788d3d0c097c3ba014658d6166fbfb81efe Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 17:25:03 +0300 Subject: [PATCH 75/97] fix(repo): address the third Step 9b.1 review round Round 3 at b3eaebaf reported two mediums, both in tests: - round 2's environment assertion accepted any directory named custody-*, which the test's own private root and destination are, so an environment built on either passed; every call's environment must now be that of the one work directory holding every home, the gpgconf query's included; - no case put a hidden or another recipient beside the pinned one, so a filter ignoring anonymous ENC_TO lines passed; both now refuse RECIPIENTS. Lows corrected, none deferred: an additional decryption subkey (capability r, GnuPG 2.4.1 and later) now refuses RECIPIENT_UNEXPECTED before encryption, checked against a real ADSK key; new cases for the finalizer record's output paths, a same-count map rename, the read-side entry count, the creation second, a prefixed status line, gpgconf beside a linked gpg's real binary, a gpg directory on PATH, an empty USTAR name, a regular-file root and a missing configuration directory; the register row, contract, JSDoc and runbook wording. A generated sweep of 182 mutants over the four modules left nine survivors, all equivalent, after five new cases; the 48 named mutants all fail a test. Refs: ADR-0111 Co-Authored-By: Claude Opus 5.5 --- docs/reference/cli/packaging.md | 7 ++- .../phase-2.5.5-hardening-and-remediation.md | 47 +++++++++++++++- docs/runbooks/local-dev-setup.md | 2 +- tools/cli-package/container.test.mjs | 29 ++++++++++ tools/cli-package/custody.mjs | 14 +++-- tools/cli-package/custody.test.mjs | 55 +++++++++++++++++-- tools/cli-package/gnupg.test.mjs | 31 ++++++++++- tools/cli-package/manifest.mjs | 4 +- tools/cli-package/ustar.test.mjs | 6 ++ 9 files changed, 174 insertions(+), 21 deletions(-) diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 15f4aadf..4c181330 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -532,8 +532,8 @@ recorded in the [Wave 3 register](../../roadmap/phases/phase-2.5.5-hardening-and and the manifest's maps as regular files of at most 16 MiB each, at most 259 entries and 128 MiB in total (the total refused before the archive is assembled). Each record must also be within the bound its own reader applies (`RECORD_BYTES`: `finalized.json` 1 MiB, - `manifest.json` and `metafile-esm.json` 4 MiB), so the container never carries a record - its own reader would refuse. Every map's digest must + `manifest.json` and `metafile-esm.json` 4 MiB), so no record exceeds the size its own + reader accepts. Every map's digest must be the manifest's, and the finalizer's record must name exactly the manifest's outputs and maps with each map's size. The manifest records no digest for the metafile, which must parse as a JSON object. The written container is read back before it is returned, so a path the @@ -568,7 +568,8 @@ recorded in the [Wave 3 register](../../roadmap/phases/phase-2.5.5-hardening-and in this order: secret material, a missing pinned primary, more than one listed key, a missing pinned subkey, revocation, expiry at the current time, unusability (invalid or disabled, no encryption capability, a creation time in the future, or an encryption subkey other than - cv25519 or RSA of at least 3072 bits) and another usable encryption subkey. GnuPG's listing + cv25519 or RSA of at least 3072 bits) and another usable encryption subkey or additional + decryption subkey (an ADSK, which GnuPG 2.4.1 and later adds as a further recipient). GnuPG's listing omits a key it cannot use, such as one created in the future or one without a valid user ID, so such a key refuses as missing or goes uncounted; GnuPG's own refusal and the `KEY_CONSIDERED` and recipient checks remain the independent second layer for it. diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 79ef5079..d018ec88 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -3703,7 +3703,7 @@ No credential values or request captures belong in tracked evidence. | 9a.2b | **Recorded split, 2026-10-24 (maintainer-approved) — product map controls and the package-test receipt:** `check:map-controls` over product copies (wrong JS by line and column, wrong map, permuted sources, wrong source, missing, malformed and truncated upstream maps, a shifted valid upstream map, and the package-test producer's post-receipt rebuild refusals that ADR-0111 lists), run in `pnpm run ci` and as a required uncached CI job inside `ci`'s needs (the required contexts stay `ci` and `coverage`); then the atomic `package_test` flip across seal, binding, consumers, aggregate and both workflows, version 1 kept. Security sitting. *Preflight decisions approved on 2026-10-24 ([decisions](#wave-3-step-9a2b-preflight-decisions--2026-10-24)); independently accepted on `development` at `0f73c253` on 2026-10-24 ([checkpoint](#wave-3-step-9a2b-implementation-checkpoint--2026-10-24), [acceptance](#wave-3-step-9a2b-acceptance--2026-10-24)).* | | NI | **Native file identity on Windows (maintainer-approved 2026-10-24; after 9a.2b, before 9b):** every file and directory identity check compares `dev`/`ino` read with `{ bigint: true }` — the filesystem policy's pinned roots, the workflow catalog's open check and the media store's pins, read check and temporary claims — so a replaced file or directory can never compare equal on Windows, where `ino` is a 64-bit NTFS file ID that a JavaScript number cannot hold exactly; a replacement test per site. Found by 9a.2a's push CI ([acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)); recorded in [deferred-tasks.md](../deferred-tasks.md#native-file-identity-on-windows--approved-increment-2026-10-24). Security sitting. *Preflight decisions approved on 2026-10-25 ([decisions](#wave-3-ni-preflight-decisions--2026-10-25)); independently accepted on `development` at `fc5af701` on 2026-10-25 ([checkpoint](#wave-3-ni-implementation-checkpoint--2026-10-25), [acceptance](#wave-3-ni-acceptance--2026-10-25)); its Windows clause closes after three green advisory Windows runs on pushed heads.* | | 9b | **Recorded split — recipient custody:** synthetic-only isolated GnuPG controls (public-only home/keyring, pinned primary and encryption-subkey fingerprints, machine-readable refusal of missing, expired, revoked or unusable recipients), encrypt once and reuse the original ciphertext. Explicit crypto/custody security sitting. Fixture proof never claims production recipient or custody configuration. *Split on 2026-10-25 (maintainer-approved, L8) into 9b.1 and 9b.2 below, which replace it as increments ([preflight decisions](#wave-3-step-9b-preflight-decisions--2026-10-25)).* | -| 9b.1 | **Recorded split, 2026-10-25 (maintainer-approved) — container, isolated GnuPG runner and encrypt-once:** the strict USTAR dialect extracted from the archive reader with its writer promoted; the private container built from the sealed staging in a staging-relative layout, each record's digest and size checked against the private manifest; an isolated public-only GnuPG runner (GnuPG ≥ 2.2.8; `--recipient-file` with no keyring or agent, and no trustdb but the empty one GnuPG may create in the listing home); a colon-listing pre-check that refuses missing, expired, revoked, unusable (v4 with cv25519 or RSA ≥ 3072 only) or unexpected recipients and secret material; `encryptOnce` (SEIPDv1 with MDC and AES-256, exactly one `ENC_TO` for the pinned subkey, the exact produced asset, a no-clobber publish, plaintext cleanup on every path); committed synthetic public fixtures, an ephemeral round-trip key, and the GnuPG prerequisite documented. Library and synthetic controls only: product paths and workflows stay unchanged. Crypto/custody security sitting. *Implemented on `development` and under review ([decisions](#wave-3-step-9b-preflight-decisions--2026-10-25), [checkpoint](#wave-3-step-9b1-implementation-checkpoint--2026-10-25)).* | +| 9b.1 | **Recorded split, 2026-10-25 (maintainer-approved) — container, isolated GnuPG runner and encrypt-once:** the strict USTAR dialect extracted from the archive reader with its writer promoted; the private container built from the sealed staging in a staging-relative layout, each map's digest checked against the private manifest and its size against the finalizer's record, which must name exactly the manifest's outputs and maps (the metafile must be a JSON object); an isolated public-only GnuPG runner (GnuPG ≥ 2.2.8; `--recipient-file` with no keyring or agent, and no trustdb but the empty one GnuPG may create in the listing home); a colon-listing pre-check that refuses missing, expired, revoked, unusable (v4 with cv25519 or RSA ≥ 3072 only) or unexpected recipients and secret material; `encryptOnce` (SEIPDv1 with MDC and AES-256, exactly one `ENC_TO` for the pinned subkey, the exact produced asset, a no-clobber publish, plaintext cleanup on every path); committed synthetic public fixtures, an ephemeral round-trip key, and the GnuPG prerequisite documented. Library and synthetic controls only: product paths and workflows stay unchanged. Crypto/custody security sitting. *Implemented on `development` and under review ([decisions](#wave-3-step-9b-preflight-decisions--2026-10-25), [checkpoint](#wave-3-step-9b1-implementation-checkpoint--2026-10-25)).* | | 9b.2 | **Recorded split, 2026-10-25 (maintainer-approved) — reuse, binding receipt and scope separation:** retained-ciphertext reuse without re-encryption (digest and size verified, zero GnuPG calls); the release binding receipt's schema v1, its validator and a sealer that writes only after the ciphertext exists (9d adds the origin, map-check and `ci`-run binding); the production recipient loader, absent in Wave 3; synthetic and production scope separation. Crypto/custody security sitting. | | 9c | **Recorded split — offline release state controller:** permission-enforcing offline GitHub/npm controller, permissioned prebuild discovery and reservation, and every rerun, partial, conflict and expiry state, with no overwrite, rebuild or re-encryption on uncertain history. | | 9d | **Recorded split — workflow, runbook and whole-Step-9 landing:** least-privilege actual workflow dependency graph, required artifact/platform aggregation and exact-artifact consumers, lost-staging refusal and the release runbook update. The whole-Step-9 cumulative review retains all 9a–9d changes with preservation traces into Step 8, and the dedicated archive/crypto security sitting closes the step. | @@ -10225,8 +10225,8 @@ Lows corrected, none deferred: - **Pins.** A pin that only converts to a fingerprint, such as a one-element array, passed. Pins must now be strings (CRIT-9B1-R2-2). - **Record bounds.** The container admitted a `manifest.json` of up to 16 MiB, which the manifest reader refuses - over 4 MiB. Each record is now held to its own reader's bound, one `RECORD_BYTES` map shared by the readers - and the container (CRIT-9B1-R2-3). + over 4 MiB. Each record is now held to its own reader's bound, one `RECORD_BYTES` map shared by the manifest + readers, the metafile read in `seal.mjs` and the container (CRIT-9B1-R2-3). - **Fixture.** The ephemeral key's home was created before its path-length check, so a failed check left it behind (T-9B1-R2-8). - **Tests.** New cases cover: @@ -10257,3 +10257,44 @@ causal-control family. On Ubuntu 24.04 with GnuPG 2.4.4 (in Docker), the 45 USTA tests ran: 44 passed and the case-insensitive-volume case skipped. The commit recording this evidence also corrects one contract phrase and changes no code. +**Round 3 at `b3eaebaf` — NONCLEAN.** Five lenses and a critic ran, every finding adversarially verified. The plan, +security and integration lenses were clean. Two mediums, both in tests, corrected: +- **The environment assertion did not name the work directory.** Round 2's check that every GnuPG call runs in the + work directory's environment accepted any directory named `custody-*`, and the test's own private root and + destination are named that way, so an environment built on either passed. Round 2's sweep had tried a leaked + process environment, never a wrong directory. Every call's environment must now be that of the one work + directory holding every home, in `mkdtemp`'s name shape, the gpgconf query's included. +- **A hidden recipient beside the pinned one was untested.** The cases covered a duplicated pinned line, a wrong + ID, a hidden ID alone and no recipient, so a filter that ignored anonymous `ENC_TO` lines passed. The pinned + subkey with a hidden or another recipient beside it now refuses `RECIPIENTS` in tests. + +Lows corrected, none deferred: +- **Additional decryption subkeys.** The pre-check did not classify an ADSK (capability `r`, GnuPG 2.4.1 and + later), which GnuPG adds as a further recipient, so a key carrying one passed it and only GnuPG's own refusal + and the recipient check stopped it. A usable ADSK now refuses `RECIPIENT_UNEXPECTED` before encryption, checked + against a real key carrying one on GnuPG 2.5.21 (COR-9B1-R3-1). +- **Tests.** New cases cover the finalizer record's output paths, a same-count map rename, the read-side entry + count one past its bound, the creation time to the second, a prefixed status line, gpgconf beside a linked + gpg's real binary, a directory named `gpg` on PATH, an empty USTAR name, a regular-file private root and a + missing configuration directory (T-9B1-R3-3 to T-9B1-R3-6, CRIT-9B1-R3-1). +- **Wording.** The 9b.1 row says which record is bound to what, round 2's record names the readers that share + `RECORD_BYTES`, the contract and its JSDoc claim only the size bound, and the runbook and a test comment no + longer say Homebrew and Ubuntu ship an empty configuration directory (PLAN-9B1-R3-1/2, CRIT-9B1-R3-2/3). + +**Sweeps.** The 48 named mutants (round 2's 30 and round 3's 18) all fail a test. A generated sweep then mutated +every one-line refusal, condition clause and comparison of `custody.mjs`, `gnupg.mjs`, `container.mjs` and +`ustar.mjs`: 182 mutants, 168 killed at first. Five survivors were real gaps and gained cases: +- a fingerprint record before any key; +- a ciphertext that can no longer be read after its listing; +- a `null` record item and a `null` finalizer record; +- a one-character USTAR name. + +The other nine are equivalent: +- two distinct status lines never share an index; +- the record items' type checks refuse with the same code at the comparison that follows; +- two sort comparators never meet a tie that matters; +- the build's map-name check refuses with the same code as the name-set check; +- the self-check's comparison cannot differ with the current writer; +- the checksum loop's extra index reads `?? 0`; +- twelve octal digits are always a safe integer. + diff --git a/docs/runbooks/local-dev-setup.md b/docs/runbooks/local-dev-setup.md index 47a44164..0c7c6fe4 100644 --- a/docs/runbooks/local-dev-setup.md +++ b/docs/runbooks/local-dev-setup.md @@ -21,7 +21,7 @@ For the full package layout this runbook references, see | Rust toolchain (`rustup`, `cargo`) | Builds the Tauri v2 desktop backend | Only needed to build `apps/desktop`. The engine, CLI, and tests do not need Rust. | | Tauri OS prerequisites | Native WebView + build deps | macOS: Xcode Command Line Tools. Windows: WebView2 + MSVC build tools. Linux: WebKitGTK + libsecret dev packages. See the [Tauri prerequisites guide](https://v2.tauri.app/start/prerequisites/). | | Git | Workflows and agents are git-native YAML | — | -| GnuPG ≥ 2.2.8 (`gpg`) | The release custody tests (`pnpm run ci`) encrypt synthetic map containers | Required, never skipped: the tests fail without it, and with a system-wide `gpg.conf` or `common.conf` in the directory `gpgconf --list-dirs sysconfdir` names (a global configuration refuses as `GPG_CONFIG`; Homebrew and Ubuntu ship that directory empty). macOS: `brew install gnupg`. Debian/Ubuntu: `apt install gnupg` (the CI Ubuntu runner has it). Windows: run these tests under WSL, since they use POSIX process and permission tools; the CI runs them on Ubuntu only. Release tooling, never a runtime dependency. | +| GnuPG ≥ 2.2.8 (`gpg`) | The release custody tests (`pnpm run ci`) encrypt synthetic map containers | Required, never skipped: the tests fail without it, and with a system-wide `gpg.conf` or `common.conf` in the directory `gpgconf --list-dirs sysconfdir` names (a global configuration refuses as `GPG_CONFIG`; Homebrew ships only `scdaemon.conf` there, and Debian and Ubuntu ship no such directory). macOS: `brew install gnupg`. Debian/Ubuntu: `apt install gnupg` (the CI Ubuntu runner has it). Windows: run these tests under WSL, since they use POSIX process and permission tools; the CI runs them on Ubuntu only. Release tooling, never a runtime dependency. | > You can do meaningful work on the engine and CLI **without** the Rust/Tauri > toolchain. Install Rust only when you start on `apps/desktop`. diff --git a/tools/cli-package/container.test.mjs b/tools/cli-package/container.test.mjs index 710bd441..d7ddceb7 100644 --- a/tools/cli-package/container.test.mjs +++ b/tools/cli-package/container.test.mjs @@ -80,6 +80,12 @@ test('a staging that is not exactly the sealed records and maps refuses', () => ['CONTAINER_SET', (copy) => writeFileSync(join(copy, 'maps', 'extra.js.map'), '{}')], ['CONTAINER_SET', (copy) => rmSync(join(copy, 'maps', mapName(copy)))], ['CONTAINER_SET', (copy) => writeFileSync(join(copy, 'maps', 'notes.txt'), '{}')], + // Same count, another valid chunk name. + [ + 'CONTAINER_SET', + (copy) => + renameSync(join(copy, 'maps', mapName(copy)), join(copy, 'maps', 'zzzrenamed.js.map')), + ], [ 'CONTAINER_DRIFT', (copy) => { @@ -96,6 +102,15 @@ test('a staging that is not exactly the sealed records and maps refuses', () => writeFileSync(path, JSON.stringify(record)); }, ], + [ + 'CONTAINER_RECORD', + (copy) => { + const path = join(copy, 'finalized.json'); + const record = JSON.parse(readFileSync(path, 'utf8')); + record.outputs[0].path = 'dist/other.js'; + writeFileSync(path, JSON.stringify(record)); + }, + ], ['CONTAINER_RECORD', (copy) => writeFileSync(join(copy, 'manifest.json'), '{"version":1}')], [ 'CONTAINER_RECORD', @@ -187,6 +202,16 @@ test('a staging that is not exactly the sealed records and maps refuses', () => writeFileSync(path, JSON.stringify(record)); }, ], + ['CONTAINER_RECORD', (copy) => writeFileSync(join(copy, 'finalized.json'), 'null')], + [ + 'CONTAINER_RECORD', + (copy) => { + const path = join(copy, 'finalized.json'); + const record = JSON.parse(readFileSync(path, 'utf8')); + record.outputs[0] = null; + writeFileSync(path, JSON.stringify(record)); + }, + ], ['CONTAINER_RECORD', (copy) => writeFileSync(join(copy, 'metafile-esm.json'), 'not json')], ['CONTAINER_RECORD', (copy) => writeFileSync(join(copy, 'metafile-esm.json'), '[]')], ]) { @@ -261,6 +286,10 @@ test('the entry, count and total bounds refuse before a container is written', ( }; assert.deepEqual(buildPrivateContainer(copy, exact).bytes, bytes); assert.equal(readPrivateContainer(bytes, exact).files.size, exact.entries); + assert.throws( + () => readPrivateContainer(bytes, { ...exact, entries: exact.entries - 1 }), + refuses('CONTAINER_FILE_COUNT'), + ); // Reading bounds each entry as building does. assert.throws( () => readPrivateContainer(bytes, { ...CONTAINER_LIMITS, entryBytes: largest - 1 }), diff --git a/tools/cli-package/custody.mjs b/tools/cli-package/custody.mjs index fbe383a7..77b85533 100644 --- a/tools/cli-package/custody.mjs +++ b/tools/cli-package/custody.mjs @@ -105,11 +105,15 @@ function keysOf(records) { const expired = (fields, now) => fields[1] === 'e' || (fields[6] !== '' && Number(fields[6]) <= now); -/** @param {string[]} fields @param {number} now */ -const usableEncryption = (fields, now) => +/** + * A usable subkey with any of `capabilities`: `e` encrypts, and `r` marks an additional decryption + * subkey (ADSK, GnuPG 2.4.1 and later), which GnuPG adds as a further recipient. + * @param {string[]} fields @param {number} now @param {string} [capabilities] + */ +const usableEncryption = (fields, now, capabilities = 'e') => !['e', 'r', 'i', 'n', 'd'].includes(fields[1]) && !expired(fields, now) && - fields[11].includes('e') && + [...capabilities].some((capability) => fields[11].includes(capability)) && !fields[11].includes('D'); /** The approved encryption-subkey algorithms: cv25519 ECDH, or RSA of at least 3072 bits. */ @@ -120,7 +124,7 @@ const approvedAlgorithm = (fields) => /** * Classify a recipient listing against its pins, in a fixed order: secret material, a missing primary, * more than one key, a missing subkey, revocation, expiry, unusability, then another usable - * encryption subkey. + * encryption or additional decryption subkey. * @param {string[][]} records colon records of `--show-keys` * @param {{primary: string, subkey: string}} pins * @param {number} now seconds since the epoch @@ -153,7 +157,7 @@ export function inspectRecipient(records, pins, now) { ) { refuseCustody('RECIPIENT_UNUSABLE'); } - if (subkeys.some((key) => key !== pinned && usableEncryption(key.fields, now))) { + if (subkeys.some((key) => key !== pinned && usableEncryption(key.fields, now, 'er'))) { refuseCustody('RECIPIENT_UNEXPECTED'); } return pins; diff --git a/tools/cli-package/custody.test.mjs b/tools/cli-package/custody.test.mjs index 1de4430d..6126516c 100644 --- a/tools/cli-package/custody.test.mjs +++ b/tools/cli-package/custody.test.mjs @@ -93,13 +93,20 @@ const valueOf = (args, flag) => args[args.indexOf(flag) + 1]; */ function recording(hooks = {}) { const calls = []; + /** @type {string | undefined} */ + let work; /** @type {typeof runCommand} */ const run = async (command, args, options) => { const kind = kindOf(args); calls.push(kind); - // Every call runs in the work directory's isolated C-locale environment, within the call bounds. - assert.deepEqual(options.env, gnupgEnvironment(options.env.HOME)); - assert.match(basename(options.env.HOME), /^custody-/u); + // Every call, the gpgconf query without a home included, runs within the call bounds in the + // isolated C-locale environment of the one work directory that holds every home. + const parent = args.includes('--homedir') ? dirname(valueOf(args, '--homedir')) : undefined; + if (kind === 'version') work = parent; + assert.ok(work !== undefined, 'the version call did not come first'); + assert.ok(parent === undefined || parent === work, 'a home outside the work directory'); + assert.match(basename(work), /^custody-[A-Za-z0-9]{6}$/u); + assert.deepEqual(options.env, gnupgEnvironment(work)); assert.equal(options.timeoutMs, GNUPG_CALL.timeoutMs); assert.equal(options.maxBytes, GNUPG_CALL.maxBytes); const hook = hooks[kind] ?? {}; @@ -209,8 +216,8 @@ test('the recipient classification refuses in its fixed order', () => { ['RECIPIENT_UNUSABLE', records({}, { 11: 'eD' })], ['RECIPIENT_UNUSABLE', records({ 1: 'n' })], ['RECIPIENT_UNUSABLE', records({ 1: 'd' })], - ['RECIPIENT_UNUSABLE', records({ 5: String(NOW + 60) })], - ['RECIPIENT_UNUSABLE', records({}, { 5: String(NOW + 60) })], + ['RECIPIENT_UNUSABLE', records({ 5: String(NOW + 1) })], + ['RECIPIENT_UNUSABLE', records({}, { 5: String(NOW + 1) })], ['RECIPIENT_UNUSABLE', records({}, { 16: 'nistp256' })], ['RECIPIENT_UNUSABLE', records({}, { 3: '1', 2: '2048', 16: '' })], // An ElGamal subkey of any size is not an approved algorithm. @@ -225,6 +232,8 @@ test('the recipient classification refuses in its fixed order', () => { { primary: PRIMARY, subkey: PRIMARY }, ], ['RECIPIENT_UNEXPECTED', [...records(), key('sub'), fpr('E'.repeat(40))]], + // An additional decryption subkey (capability `r`) is a further recipient GnuPG adds. + ['RECIPIENT_UNEXPECTED', [...records(), key('sub', { 11: 'r' }), fpr('E'.repeat(40))]], ]) { assert.throws( () => inspectRecipient(listing, override ?? pins, NOW), @@ -251,6 +260,19 @@ test('the recipient classification refuses in its fixed order', () => { pins, ); assert.equal(inspectRecipient(records({}, { 3: '1', 2: '4096', 16: '' }), pins, NOW), pins); + // A fingerprint record before any key is ignored. + assert.equal(inspectRecipient([fpr(PRIMARY), ...records()], pins, NOW), pins); + // Created in the current second is not in the future; a revoked additional decryption subkey is + // not a recipient. + assert.equal(inspectRecipient(records({ 5: String(NOW) }, { 5: String(NOW) }), pins, NOW), pins); + assert.equal( + inspectRecipient( + [...records(), key('sub', { 1: 'r', 11: 'r' }), fpr('E'.repeat(40))], + pins, + NOW, + ), + pins, + ); }); test('the committed synthetic fixtures list as their names say under the real GnuPG', async () => { @@ -448,6 +470,11 @@ test('the ciphertext must name exactly the pinned subkey and nothing else', asyn (text) => text.replace(/ENC_TO [0-9A-F]{16}/u, `ENC_TO ${'1'.repeat(16)}`), (text) => text.replace(/ENC_TO [0-9A-F]{16}/u, `ENC_TO ${'0'.repeat(16)}`), (text) => text.replace(/\[GNUPG:\] ENC_TO[^\n]*\n/u, ''), + // The pinned subkey with a hidden or another recipient beside it. + (text) => + text.replace(/(\[GNUPG:\] ENC_TO[^\n]*\n)/u, `$1[GNUPG:] ENC_TO ${'0'.repeat(16)} 18 0\n`), + (text) => + text.replace(/(\[GNUPG:\] ENC_TO[^\n]*\n)/u, `$1[GNUPG:] ENC_TO ${'1'.repeat(16)} 18 0\n`), ]) { refused(await attempt(recording(rewrite(change))), 'RECIPIENTS'); } @@ -568,7 +595,8 @@ test("a global GnuPG configuration in the installation's sysconfdir refuses befo const sysconf = fresh('sysconfdir'); mkdirSync(sysconf); const config = (stdout) => ({ config: { replace: () => ({ stdout, stderr: '' }) } }); - // An empty configuration directory, as Homebrew and Ubuntu ship it, admits. + // A configuration directory with neither file admits: Homebrew ships only `scdaemon.conf` there, + // and Debian and Ubuntu ship no such directory. const admitted = recording(config(`${sysconf}\n`)); const outcome = await attempt(admitted); assert.equal(outcome.error, undefined, String(outcome.error)); @@ -610,6 +638,16 @@ test('the published bytes are the listed bytes, and the default clock counts sec recipients: { before: (args) => writeFileSync(args.at(-1), other) }, }); refused(await attempt(swapped), 'ASSET'); + // One that can no longer be read after its listing refuses the same way. + const vanished = recording({ + recipients: { + after: (result, args) => { + rmSync(args.at(-1)); + return result; + }, + }, + }); + refused(await attempt(vanished), 'ASSET'); // A recipient whose subkey expires tomorrow is valid under the default clock (seconds since the // epoch); a clock in milliseconds would call it expired. const pins = PINS.good; @@ -816,9 +854,14 @@ test('an overlap spelled differently on a case-insensitive volume still refuses' test('a missing destination or private root refuses with a fixed code and no path', async () => { const { root, destination } = places(); const missing = join(fresh('missing'), 'nowhere'); + // An owner-only regular file is not a private root. + const file = join(fresh('file-root'), 'nowhere'); + mkdirSync(dirname(file)); + writeFileSync(file, '', { mode: 0o600 }); for (const [code, overrides] of [ ['PLACEMENT', { destination: missing, privateRoot: root }], ['PRIVATE_ROOT', { destination, privateRoot: missing }], + ['PRIVATE_ROOT', { destination, privateRoot: file }], ]) { await assert.rejects( encryptOnce({ diff --git a/tools/cli-package/gnupg.test.mjs b/tools/cli-package/gnupg.test.mjs index 3b49366e..14633b9e 100644 --- a/tools/cli-package/gnupg.test.mjs +++ b/tools/cli-package/gnupg.test.mjs @@ -1,5 +1,13 @@ import assert from 'node:assert/strict'; -import { mkdirSync, mkdtempSync, rmSync, statSync, symlinkSync, writeFileSync } from 'node:fs'; +import { + mkdirSync, + mkdtempSync, + realpathSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import { delimiter, dirname, isAbsolute, join, relative } from 'node:path'; import { after, test } from 'node:test'; @@ -49,6 +57,24 @@ test('gpg resolves from absolute PATH entries only and is required, never skippe refuses('GPG_UNAVAILABLE'), ); assert.deepEqual(resolveGpg(join(scratch, 'relative')).prefix, []); + // A directory named gpg is skipped, never chosen. + const directoryEntry = createGnupgHome(scratch, 'gpg-directory'); + mkdirSync(join(directoryEntry, 'gpg')); + assert.equal( + resolveGpg([directoryEntry, dirname(gpg.command)].join(delimiter)).command, + gpg.command, + ); +}); + +test("gpgconf is the one beside the linked gpg's real binary, never a neighbour of the link", () => { + const real = resolveGpg(); + const farm = createGnupgHome(scratch, 'link-farm'); + symlinkSync(real.command, join(farm, 'gpg')); + writeFileSync(join(farm, 'gpgconf'), '#!/bin/sh\necho decoy\n', { mode: 0o700 }); + const linked = resolveGpg(farm); + assert.equal(linked.command, realpathSync(join(farm, 'gpg'))); + assert.equal(linked.gpgconf.command, join(dirname(linked.command), 'gpgconf')); + assert.notEqual(dirname(linked.gpgconf.command), farm); }); test('the version floor is GnuPG 2.2.8, read from the version line only', async () => { @@ -196,6 +222,8 @@ test('a global configuration in the sysconfdir gpgconf names refuses', async () const other = createGnupgHome(scratch, 'sysconf-other'); writeFileSync(join(other, 'scdaemon.conf'), 'disable-ccid\n'); await check(`${other}\n`); + // Debian and Ubuntu name a directory they do not ship: nothing there to read. + await check(`${join(scratch, 'sysconf-missing')}\n`); // A named directory is printed raw: a literal `%41` is part of the path, never an escape. const raw = createGnupgHome(scratch, 'sysconf %41'); writeFileSync(join(raw, 'gpg.conf'), 'encrypt-to 0123456789ABCDEF\n'); @@ -228,6 +256,7 @@ test('status parsing admits only status lines and keeps unknown keywords for the '[GNUPG:] begin\n', '[GNUPG:] BEGIN_ENCRYPTION 2 9\n\n', '[GNUPG:]BEGIN_ENCRYPTION\n', + 'gpg: noise [GNUPG:] NEWSIG\n', ]) { assert.throws(() => parseStatus(text), refuses('STATUS'), text); } diff --git a/tools/cli-package/manifest.mjs b/tools/cli-package/manifest.mjs index 0edc0143..b9a91af0 100644 --- a/tools/cli-package/manifest.mjs +++ b/tools/cli-package/manifest.mjs @@ -51,8 +51,8 @@ const SOURCE = new RegExp( const VERSION = /^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]{1,64})?$/u; const LIMITS = Object.freeze({ outputs: 256, sources: 8192, recordBytes: 1024 * 1024 }); /** - * The bound each staged record's reader applies, by file name: the custody container carries no record - * its reader would refuse. + * The size bound each staged record's reader applies, by file name: no record in the custody container + * exceeds the size its reader accepts. */ export const RECORD_BYTES = Object.freeze({ 'finalized.json': LIMITS.recordBytes, diff --git a/tools/cli-package/ustar.test.mjs b/tools/cli-package/ustar.test.mjs index a87b269c..df1c586d 100644 --- a/tools/cli-package/ustar.test.mjs +++ b/tools/cli-package/ustar.test.mjs @@ -99,6 +99,7 @@ test('each header field outside the dialect refuses with its own code', () => { ['TAR_FORMAT', mutated((block) => block.write('xx', 263))], ['TAR_NUMBER', mutated((block) => block.write('00000000009 ', 124))], ['TAR_NAME', mutated((block) => (block[20] = 0x61))], + ['TAR_NAME', mutated((block) => block.fill(0, 0, 100))], ['UTF8', mutated((block) => block.fill(0xff, 0, 4))], ['TAR_MODE', mutated((block) => block.write('000644', 100))], ['FILE_SIZE', mutated((block) => block.write('00000010001 ', 124))], @@ -134,6 +135,11 @@ test('the archive structure refuses duplicates, aliases, truncation, padding, co ]) { assert.throws(() => read(archive), refuses(code), code); } + // A one-character name is a name: the caller's path rule judges it. + assert.throws( + () => read(Buffer.concat([ustarEntry('a', Buffer.from('{}'), 0o600), end])), + refuses('PATH'), + ); // The caller's path rule runs inside the header check, before its mode and size are judged. assert.throws( () => read(Buffer.concat([ustarEntry('outside.txt', Buffer.alloc(8192), 0o644), end])), From 56ccae62754b623fd1dae0e1d6eff3aac268681a Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 19:27:57 +0300 Subject: [PATCH 76/97] docs(repo): record the Step 9b.1 acceptance Round 4 at 147c2788, a new complete cumulative review of 12883df4..147c2788 with every reviewer in its own worktree, reported no blocker, high or medium finding and no regression, and its separate crypto/custody security sitting was clean. The full pnpm run ci passed plain and alone at that head (test:acceptance-tools 980 of 980, all 25 product map controls, every causal-control family), and the custody tests passed on Ubuntu 24.04 with GnuPG 2.4.4. Four lows are recorded in deferred-tasks.md: a temporary sibling that cannot be removed after the link reports UNEXPECTED; no test pins the metafile's 4 MiB value; no test builds the container from a real finalizer's staging; the overlap check misses containment through a bind mount. The fifth (the Windows image's second gpg) and the contract's overlap sentence are corrected here. This commit changes no code. Step 9b.2 is next. Refs: ADR-0111 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 +- CLAUDE.md | 4 +- docs/reference/cli/README.md | 2 +- docs/reference/cli/packaging.md | 13 +++--- docs/roadmap/current.md | 6 +-- docs/roadmap/deferred-tasks.md | 45 +++++++++++++++++-- .../phase-2.5.5-hardening-and-remediation.md | 37 ++++++++++++++- 7 files changed, 93 insertions(+), 18 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f221495a..362b9580 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,7 +117,7 @@ explicit; source CI does not certify later documentation commits. The joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now -accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI is accepted too, Step 9b.1 is implemented and under review, and Steps 9b.2–19 have not started) and whole-wave acceptance remains Step 19. +accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI and Step 9b.1 are accepted too, and Steps 9b.2–19 have not started) and whole-wave acceptance remains Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification @@ -130,7 +130,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25): round 1 reported no blocker, high or medium finding and no regression, with a clean security sitting; every product file and directory identity check now compares exact bigints, its low findings are recorded in deferred-tasks.md, and its Windows clause closes after three green advisory Windows runs ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)). Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b-preflight-decisions--2026-10-25)); Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-implementation-checkpoint--2026-10-25)). `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25): round 1 reported no blocker, high or medium finding and no regression, with a clean security sitting; every product file and directory identity check now compares exact bigints, its low findings are recorded in deferred-tasks.md, and its Windows clause closes after three green advisory Windows runs ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)). Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b-preflight-decisions--2026-10-25)); Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is independently accepted on `development` at `147c2788` (2026-10-25): rounds 1–3 were not accepted (one, one and two mediums, each corrected), and round 4 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-acceptance--2026-10-25)). Step 9b.2 is next. `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. See [docs/roadmap/current.md](docs/roadmap/current.md) for live status. diff --git a/CLAUDE.md b/CLAUDE.md index 653fa6ca..1c958286 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -206,7 +206,7 @@ explicit; source CI does not certify later documentation commits. The joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now -accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI is accepted too, Step 9b.1 is implemented and under review, and Steps 9b.2–19 have not started) and whole-wave acceptance remains Step 19. +accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI and Step 9b.1 are accepted too, and Steps 9b.2–19 have not started) and whole-wave acceptance remains Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification @@ -219,7 +219,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25): round 1 reported no blocker, high or medium finding and no regression, with a clean security sitting; every product file and directory identity check now compares exact bigints, its low findings are recorded in deferred-tasks.md, and its Windows clause closes after three green advisory Windows runs ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)). Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b-preflight-decisions--2026-10-25)); Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is implemented on `development` and under review ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-implementation-checkpoint--2026-10-25)). `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25): round 1 reported no blocker, high or medium finding and no regression, with a clean security sitting; every product file and directory identity check now compares exact bigints, its low findings are recorded in deferred-tasks.md, and its Windows clause closes after three green advisory Windows runs ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)). Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b-preflight-decisions--2026-10-25)); Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is independently accepted on `development` at `147c2788` (2026-10-25): rounds 1–3 were not accepted (one, one and two mediums, each corrected), and round 4 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-acceptance--2026-10-25)). Step 9b.2 is next. `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. For live status, per-PR history, milestone dates, and open obligations, see the canonical diff --git a/docs/reference/cli/README.md b/docs/reference/cli/README.md index 639dfe27..a0ad0da1 100644 --- a/docs/reference/cli/README.md +++ b/docs/reference/cli/README.md @@ -13,5 +13,5 @@ Part of [reference/](../README.md). | [commands.md](commands.md) | The full `relavium` command surface (run, chat, chat-resume/list/export, agent run, list, create, import, export, logs, gate, init, provider) plus interactive TUI vs CI JSON mode. | | [home.md](home.md) | The bare-invocation interactive **Home** — the TTY gate, the read-only management strip over `history.db`, the Home↔chat mode machine + keymap, bracketed paste, the min-terminal degrade, and the signal/exit-code lifecycle (2.5.B / ADR-0054). | | [chat-session.md](chat-session.md) | The `relavium chat` agent-session REPL — entry, agent/model selection, the multi-turn loop, streaming, tool availability, the `--json` session-event stream, and exit code 4. | -| [packaging.md](packaging.md) | Wave 3 finite runtime graph, installed-package consumers and Step 8 native activation accepted on `development`; the Step 9a.1 candidate build and finalizer are accepted on `development` and Step 9a.2a's private manifest and symbolication are accepted on `development` and Step 9a.2b's product map controls and `package_test` receipt are accepted on `development`; Step 9b.1's private container, isolated GnuPG runner and encrypt-once library are implemented on `development` and under review (synthetic recipients only); the remaining Step 9 custody and release gates remain staged. | +| [packaging.md](packaging.md) | Wave 3 finite runtime graph, installed-package consumers and Step 8 native activation accepted on `development`; the Step 9a.1 candidate build and finalizer are accepted on `development` and Step 9a.2a's private manifest and symbolication are accepted on `development` and Step 9a.2b's product map controls and `package_test` receipt are accepted on `development`; Step 9b.1's private container, isolated GnuPG runner and encrypt-once library are accepted on `development` (synthetic recipients only); the remaining Step 9 custody and release gates remain staged. | | [agent-run-fixture.md](agent-run-fixture.md) | The `relavium agent run --fixture` cassette format — a `StreamChunk[][]` JSON recording of an agent run's LLM stream, for deterministic offline replay. | diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 4c181330..2d829bd9 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -1,6 +1,6 @@ # CLI Packaging Contract -- **Status**: Compiled graph, installed package and Step 8 native activation independently accepted on `development`; the Step 9a.1 candidate build and finalizer are independently accepted on `development` (2026-10-24); Step 9a.2a's strict decoder, private manifest and symbolication are independently accepted on `development` (2026-10-24); Step 9a.2b's product map controls, post-receipt rebuild refusals and `package_test` receipt are independently accepted on `development` (2026-10-24); Step 9b.1's private container, isolated GnuPG runner and encrypt-once library are implemented on `development` and under review (synthetic recipients only); the remaining Step 9 custody and release gates remain staged +- **Status**: Compiled graph, installed package and Step 8 native activation independently accepted on `development`; the Step 9a.1 candidate build and finalizer are independently accepted on `development` (2026-10-24); Step 9a.2a's strict decoder, private manifest and symbolication are independently accepted on `development` (2026-10-24); Step 9a.2b's product map controls, post-receipt rebuild refusals and `package_test` receipt are independently accepted on `development` (2026-10-24); Step 9b.1's private container, isolated GnuPG runner and encrypt-once library are independently accepted on `development` (2026-10-25; synthetic recipients only); the remaining Step 9 custody and release gates remain staged - **Date**: 2026-10-15 - **Related**: [ADR-0111](../../decisions/0111-cli-meta-operations-use-a-lazy-packaged-runtime.md), [commands.md](commands.md), [release runbook](../../runbooks/release-a-surface.md) @@ -575,8 +575,10 @@ recorded in the [Wave 3 register](../../roadmap/phases/phase-2.5.5-hardening-and `KEY_CONSIDERED` and recipient checks remain the independent second layer for it. - **Encrypt once.** `encryptOnce` refuses before any GnuPG call when the destination already holds `relavium--private-maps.tar.gpg`, when the destination and the owner-only - private root overlap (compared by device and inode, so a second spelling of one directory - does not hide the overlap), or when either is missing or the root is not an owner-only + private root overlap (compared by device and inode, so one directory reached by a second + spelling, such as a case-insensitive variant or a bind mount of that directory, does not hide + the overlap; containment through a bind mount of a directory inside the other is not detected, + a [recorded limit](../../roadmap/deferred-tasks.md#wave-3-step-9b1--recorded-low-findings-2026-10-25)), or when either is missing or the root is not an owner-only directory of this user. No root qualifies on Windows, where modes say nothing about access and no ACL check exists. Success requires exit 0, exactly one `KEY_CONSIDERED` for the pinned primary, one `BEGIN_ENCRYPTION 2 9` (SEIPDv1 with MDC and @@ -608,8 +610,9 @@ recorded in the [Wave 3 register](../../roadmap/phases/phase-2.5.5-hardening-and `pnpm run ci` and the four Ubuntu jobs inside `ci`'s needs, against the runner's GnuPG; they fail, never skip, without GnuPG ≥ 2.2.8 or with a global configuration ([local-dev-setup.md](../../runbooks/local-dev-setup.md)). They do not run - on Windows: no Windows job runs `test:acceptance-tools`, the image's only `gpg` is Git for - Windows' MSYS2 build, never exercised here, and the tests use POSIX process and permission + on Windows: no Windows job runs `test:acceptance-tools`, the `gpg` on the image's PATH is + Git for Windows' MSYS2 build and MSYS2's own GnuPG under `C:\msys64` is installed off PATH, + neither exercised here, and the tests use POSIX process and permission tools; `encryptOnce` itself refuses there. The 2.2.8 floor is enforced by the version check without being exercised. diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index bacbcb92..45adfc0e 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -138,7 +138,7 @@ The [deferred register](deferred-tasks.md) retains its existing limits; closure **Wave 3 Steps 1–8 merged to `main` on 2026-10-18 as [PR #95](https://github.com/HodeTech/Relavium/pull/95), merge commit `6d2a7372` (22:44 UTC). Wave 3 itself is not complete: Steps 9–19 had not started at -that merge (Steps 9a.1, 9a.2a and 9a.2b are now accepted on `development`, the native-identity increment NI is accepted too, Step 9b.1 is implemented and under review, and Steps 9b.2–19 have not started) and whole-wave acceptance +that merge (Steps 9a.1, 9a.2a and 9a.2b are now accepted on `development`, the native-identity increment NI and Step 9b.1 are accepted too, and Steps 9b.2–19 have not started) and whole-wave acceptance remains Step 19.** The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) joins the completed scope, every review attribution, the independent review's dispositions and the publication state observed at merge. It was the maintainer-authorized, scoped publication checkpoint. @@ -172,7 +172,7 @@ bar and round cap, as its medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, by the author alone). Its other findings are recorded in -[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is independently accepted on `development` at `cf97b7f1` (2026-10-24): its round 13 reported no blocker, high or medium finding and its security sitting was clean, and the round's low findings are recorded in deferred-tasks.md ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)); Step 9a.2 is split into 9a.2a and 9a.2b, and Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24) after three rounds, the third with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)); Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24) after two rounds, the second with no blocker, high or medium finding and no regression ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)), and the native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25) after one round with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)), its Windows clause open until three green advisory Windows runs; Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b-preflight-decisions--2026-10-25)), and Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is implemented on `development` and under review ([checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-implementation-checkpoint--2026-10-25)); the substrate's +[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is independently accepted on `development` at `cf97b7f1` (2026-10-24): its round 13 reported no blocker, high or medium finding and its security sitting was clean, and the round's low findings are recorded in deferred-tasks.md ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)); Step 9a.2 is split into 9a.2a and 9a.2b, and Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24) after three rounds, the third with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)); Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24) after two rounds, the second with no blocker, high or medium finding and no regression ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)), and the native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25) after one round with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)), its Windows clause open until three green advisory Windows runs; Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b-preflight-decisions--2026-10-25)), and Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is independently accepted on `development` at `147c2788` ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-acceptance--2026-10-25)), with Step 9b.2 next; the substrate's 9a.1–9d increments carry a dedicated archive/crypto security sitting. Steps 10–19 follow the [increment table](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) and end with whole-wave acceptance at Step 19. `development` is pushed after each accepted increment; PR #95 no longer bounds the @@ -1141,7 +1141,7 @@ an ESLint selector rather than a list. ### Wave 3 — Clear the ground -**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Steps 9a.1, 9a.2a, 9a.2b and NI accepted on `development` (2026-10-24/25), Step 9b.1 implemented and under review, Steps 9b.2–19 NOT STARTED — 2026-10-25.** +**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Steps 9a.1, 9a.2a, 9a.2b, NI and 9b.1 accepted on `development` (2026-10-24/25), Steps 9b.2–19 NOT STARTED — 2026-10-25.** ADR-0110–0113, their linked contracts and the nineteen-increment plan were approved on 2026-10-16. [Steps 1–8 were independently accepted](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) on `development`, with whole Step 8 reviewed at `c83c8575`, and published as the scoped checkpoint diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index bfd3e183..848586c3 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -2,7 +2,7 @@ > Status: Living -> Last updated: 2026-10-25 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Step 9a.1 is independently accepted on `development` with its recorded low findings below, Step 9a.2a is independently accepted on `development` with its recorded low findings below, Step 9a.2b is independently accepted on `development` with its recorded low findings below, the native-identity increment NI independently accepted on `development` with its recorded low findings below (its Windows clause open), Step 9b.1 implemented and under review with its named limits below, and Steps 9b.2–19 not started; the PR95 increment is closed on `development` under the maintainer's 2026-10-21 round cap). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's edge cases with the maintainer's 2026-10-21 acceptance bar and round cap and the 2026-10-22 rulings. Existing approved deferrals remain. +> Last updated: 2026-10-25 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Step 9a.1 is independently accepted on `development` with its recorded low findings below, Step 9a.2a is independently accepted on `development` with its recorded low findings below, Step 9a.2b is independently accepted on `development` with its recorded low findings below, the native-identity increment NI independently accepted on `development` with its recorded low findings below (its Windows clause open), Step 9b.1 independently accepted on `development` with its named limits and recorded low findings below, and Steps 9b.2–19 not started; the PR95 increment is closed on `development` under the maintainer's 2026-10-21 round cap). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's edge cases with the maintainer's 2026-10-21 acceptance bar and round cap and the 2026-10-22 rulings. Existing approved deferrals remain. > The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) records completed prerequisites; full M/N lifecycle and other scheduled work remain with their owners. > > Earlier triage: the Phase-2.6 rewrite triaged every open item, the 2026-07-19 full-project @@ -787,13 +787,14 @@ Recorded with the Step 9b.1 implementation **Trigger for every entry:** the next change to the file it names, or a release producer on that platform. - [ ] **The custody tests do not run on Windows, and `encryptOnce` refuses there.** *Low; a named limit.* - No Windows job runs `test:acceptance-tools`; the Windows image's only `gpg` is Git for Windows' MSYS2 - build in `usr\bin`, never exercised here, and the tests use POSIX process and permission tools (`ps`, + No Windows job runs `test:acceptance-tools`; the `gpg` on the Windows image's PATH is Git for Windows' + MSYS2 build in `usr\bin`, and MSYS2's own GnuPG under `C:\msys64` is installed off PATH, neither exercised + here, and the tests use POSIX process and permission tools (`ps`, owner-only modes), so `docs/runbooks/local-dev-setup.md` sends Windows developers to WSL. Windows modes say nothing about access and no ACL check exists, so since round 2 (SEC-9B1-R2-2) no private root qualifies there and every encryption refuses `PRIVATE_ROOT` before any GnuPG call. **Re-check:** before a release producer runs on Windows, add an ACL-based owner-only check for the private root and its work - directory, then run the custody tests on that runner and record which `gpg.exe` `resolveGpg` selects + directory, then run the custody tests on that runner and record which of the two `gpg.exe` builds `resolveGpg` selects and whether its Windows path handling and gpgconf's line ending pass. *(low · tools/cli-package/custody.mjs, tools/cli-package/gnupg.mjs)* - [ ] **An interrupted custody test run can leave its ephemeral key home.** *Low; test-only.* *(rigor pass, @@ -826,6 +827,42 @@ Recorded with the Step 9b.1 implementation against GnuPG 2.2.8 (for example in a container that ships it) before a recovery runbook names that version. *(low · tools/cli-package/gnupg.mjs)* +## Wave 3 Step 9b.1 — recorded low findings (2026-10-25) + +Recorded with the Step 9b.1 acceptance after round 4 +([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-acceptance--2026-10-25)). +**Trigger for every entry:** the next change to the file it names, or Step 9d's release integration. + +- [ ] **A temporary sibling that cannot be removed after the link reports `UNEXPECTED`.** *Low; round 4, + COR-9B1-R4-1.* `publish` removes its `.partial` sibling in a `finally`. If that removal fails (`EACCES`, + `EROFS`, `EIO`) after `linkSync` succeeded, `encryptOnce` refuses `UNEXPECTED` with no record although the + ciphertext is published; the sibling, ciphertext only and never plaintext, stays beside it, and a retry + refuses `ALREADY_ENCRYPTED`. Example: make the destination 0500 right after the link. **Re-check:** make the + removal best-effort once the link succeeded, or refuse with a code that says the ciphertext was published, and + inject that failure in a test alongside T-9B1-R1-3's. *(low · tools/cli-package/custody.mjs)* +- [ ] **No test pins the metafile's 4 MiB bound.** *Low; round 4, T-9B1-R4-1.* The container's record-bound + tests pad from `RECORD_BYTES` itself, so changing `'metafile-esm.json'` to 8 MiB passes every test while the + contract says 4 MiB; `manifest.test.mjs` pins the other two values. The container and `seal.mjs` share the + constant, so every record still stays within its reader's bound. **Re-check:** assert `RECORD_BYTES`'s literal + values, or pad the metafile to a literal 4 MiB plus one byte. *(low · tools/cli-package/manifest.mjs)* +- [ ] **No test builds the container from a real finalizer's staging.** *Low; round 4, INT-9B1-R4-1.* The + container requires the finalizer record's exact key sets, but every container and custody test uses + `candidate-fixture.mjs`'s hand-written record and a stub metafile. A field added to `finalize.mjs`'s record + would pass every gate and refuse `CONTAINER_RECORD` on every real staging. A reviewer's probe built, + encrypted, decrypted and extracted a real production's staging at `147c2788` without a refusal. **Re-check:** + build and read back the container once from a real production's sealed staging (map-controls' seal port holds + one), or pin the record's key set in `finalize.test.mjs`; at the latest in Step 9d. *(low · + tools/cli-package/container.mjs, tools/cli-package/finalize.mjs)* +- [ ] **The overlap check misses containment through a bind mount.** *Low; round 4, CRIT-9B1-R4-1.* `within` + compares device and inode along each path's own ancestors. It catches one directory reached by two spellings + (a case-insensitive variant, or a bind mount of exactly that directory), but not a root that is a bind mount + of a directory inside the destination, or the reverse: the work directory and its plaintext then sit inside + the destination's tree while GnuPG runs. A bind mount needs privileges to set up. The acceptance commit + narrows the contract's sentence; `custody.mjs`'s comment still says a bind mount can give one directory two + spellings, without the limit. **Re-check:** in Step 9d, create the root and destination as fresh sibling + directories of the job, or compare against the mount table, before relying on the refusal; narrow the comment + on the next change to `custody.mjs`. *(low · tools/cli-package/custody.mjs)* + ## Wave 3 NI — recorded low findings (2026-10-25) Round 1 of the native-identity increment at `fc5af701` accepted it with these lows diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index d018ec88..edadf53d 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -3703,7 +3703,7 @@ No credential values or request captures belong in tracked evidence. | 9a.2b | **Recorded split, 2026-10-24 (maintainer-approved) — product map controls and the package-test receipt:** `check:map-controls` over product copies (wrong JS by line and column, wrong map, permuted sources, wrong source, missing, malformed and truncated upstream maps, a shifted valid upstream map, and the package-test producer's post-receipt rebuild refusals that ADR-0111 lists), run in `pnpm run ci` and as a required uncached CI job inside `ci`'s needs (the required contexts stay `ci` and `coverage`); then the atomic `package_test` flip across seal, binding, consumers, aggregate and both workflows, version 1 kept. Security sitting. *Preflight decisions approved on 2026-10-24 ([decisions](#wave-3-step-9a2b-preflight-decisions--2026-10-24)); independently accepted on `development` at `0f73c253` on 2026-10-24 ([checkpoint](#wave-3-step-9a2b-implementation-checkpoint--2026-10-24), [acceptance](#wave-3-step-9a2b-acceptance--2026-10-24)).* | | NI | **Native file identity on Windows (maintainer-approved 2026-10-24; after 9a.2b, before 9b):** every file and directory identity check compares `dev`/`ino` read with `{ bigint: true }` — the filesystem policy's pinned roots, the workflow catalog's open check and the media store's pins, read check and temporary claims — so a replaced file or directory can never compare equal on Windows, where `ino` is a 64-bit NTFS file ID that a JavaScript number cannot hold exactly; a replacement test per site. Found by 9a.2a's push CI ([acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)); recorded in [deferred-tasks.md](../deferred-tasks.md#native-file-identity-on-windows--approved-increment-2026-10-24). Security sitting. *Preflight decisions approved on 2026-10-25 ([decisions](#wave-3-ni-preflight-decisions--2026-10-25)); independently accepted on `development` at `fc5af701` on 2026-10-25 ([checkpoint](#wave-3-ni-implementation-checkpoint--2026-10-25), [acceptance](#wave-3-ni-acceptance--2026-10-25)); its Windows clause closes after three green advisory Windows runs on pushed heads.* | | 9b | **Recorded split — recipient custody:** synthetic-only isolated GnuPG controls (public-only home/keyring, pinned primary and encryption-subkey fingerprints, machine-readable refusal of missing, expired, revoked or unusable recipients), encrypt once and reuse the original ciphertext. Explicit crypto/custody security sitting. Fixture proof never claims production recipient or custody configuration. *Split on 2026-10-25 (maintainer-approved, L8) into 9b.1 and 9b.2 below, which replace it as increments ([preflight decisions](#wave-3-step-9b-preflight-decisions--2026-10-25)).* | -| 9b.1 | **Recorded split, 2026-10-25 (maintainer-approved) — container, isolated GnuPG runner and encrypt-once:** the strict USTAR dialect extracted from the archive reader with its writer promoted; the private container built from the sealed staging in a staging-relative layout, each map's digest checked against the private manifest and its size against the finalizer's record, which must name exactly the manifest's outputs and maps (the metafile must be a JSON object); an isolated public-only GnuPG runner (GnuPG ≥ 2.2.8; `--recipient-file` with no keyring or agent, and no trustdb but the empty one GnuPG may create in the listing home); a colon-listing pre-check that refuses missing, expired, revoked, unusable (v4 with cv25519 or RSA ≥ 3072 only) or unexpected recipients and secret material; `encryptOnce` (SEIPDv1 with MDC and AES-256, exactly one `ENC_TO` for the pinned subkey, the exact produced asset, a no-clobber publish, plaintext cleanup on every path); committed synthetic public fixtures, an ephemeral round-trip key, and the GnuPG prerequisite documented. Library and synthetic controls only: product paths and workflows stay unchanged. Crypto/custody security sitting. *Implemented on `development` and under review ([decisions](#wave-3-step-9b-preflight-decisions--2026-10-25), [checkpoint](#wave-3-step-9b1-implementation-checkpoint--2026-10-25)).* | +| 9b.1 | **Recorded split, 2026-10-25 (maintainer-approved) — container, isolated GnuPG runner and encrypt-once:** the strict USTAR dialect extracted from the archive reader with its writer promoted; the private container built from the sealed staging in a staging-relative layout, each map's digest checked against the private manifest and its size against the finalizer's record, which must name exactly the manifest's outputs and maps (the metafile must be a JSON object); an isolated public-only GnuPG runner (GnuPG ≥ 2.2.8; `--recipient-file` with no keyring or agent, and no trustdb but the empty one GnuPG may create in the listing home); a colon-listing pre-check that refuses missing, expired, revoked, unusable (v4 with cv25519 or RSA ≥ 3072 only) or unexpected recipients and secret material; `encryptOnce` (SEIPDv1 with MDC and AES-256, exactly one `ENC_TO` for the pinned subkey, the exact produced asset, a no-clobber publish, plaintext cleanup on every path); committed synthetic public fixtures, an ephemeral round-trip key, and the GnuPG prerequisite documented. Library and synthetic controls only: product paths and workflows stay unchanged. Crypto/custody security sitting. *Independently accepted on `development` at `147c2788` (2026-10-25) ([decisions](#wave-3-step-9b-preflight-decisions--2026-10-25), [checkpoint](#wave-3-step-9b1-implementation-checkpoint--2026-10-25), [acceptance](#wave-3-step-9b1-acceptance--2026-10-25)).* | | 9b.2 | **Recorded split, 2026-10-25 (maintainer-approved) — reuse, binding receipt and scope separation:** retained-ciphertext reuse without re-encryption (digest and size verified, zero GnuPG calls); the release binding receipt's schema v1, its validator and a sealer that writes only after the ciphertext exists (9d adds the origin, map-check and `ci`-run binding); the production recipient loader, absent in Wave 3; synthetic and production scope separation. Crypto/custody security sitting. | | 9c | **Recorded split — offline release state controller:** permission-enforcing offline GitHub/npm controller, permissioned prebuild discovery and reservation, and every rerun, partial, conflict and expiry state, with no overwrite, rebuild or re-encryption on uncertain history. | | 9d | **Recorded split — workflow, runbook and whole-Step-9 landing:** least-privilege actual workflow dependency graph, required artifact/platform aggregation and exact-artifact consumers, lost-staging refusal and the release runbook update. The whole-Step-9 cumulative review retains all 9a–9d changes with preservation traces into Step 8, and the dedicated archive/crypto security sitting closes the step. | @@ -10298,3 +10298,38 @@ The other nine are equivalent: - the checksum loop's extra index reads `?? 0`; - twelve octal digits are always a safe integer. + +### Wave 3 Step 9b.1 acceptance — 2026-10-25 + +**Step 9b.1 is independently accepted on `development` at `147c2788` (2026-10-25)** under the Acceptance row as +amended on 2026-10-23. Rounds 1–3 were not accepted, each with mediums corrected as the +[checkpoint](#wave-3-step-9b1-implementation-checkpoint--2026-10-25) records: +- round 1, a global GnuPG configuration the "no options file" claim missed; +- round 2, round 1's correction left the outer `CLEANUP` override untested; +- round 3, round 2's environment assertion accepted the wrong directory, and no case put a hidden recipient beside + the pinned one. + +Round 4 was a NEW independent complete cumulative review of `12883df4..147c2788`, with every reviewer in its own +worktree detached at that head. It reported no blocker, high or medium finding and no regression. Its separate +crypto/custody security sitting, run at the same head as its own reviewer and report, was clean. The plan lens was +clean too. It mapped every clause of the 9b.1 row and rulings L1–L8, C1 and C2 to code, found the inventory pin and +every import valid at all twelve commits, and confirmed that nothing outside the tests imports the library. + +**Author evidence at `147c2788`.** The full `pnpm run ci` passed plain and alone (exit 0): `test:acceptance-tools` +980 of 980, all 25 product map controls, and every causal-control family. On Ubuntu 24.04 with GnuPG 2.4.4 (in +Docker), the 46 USTAR, container, GnuPG and custody tests ran: 45 passed and the case-insensitive-volume case +skipped. + +The round's verified findings were five lows: +- Four are recorded in + [deferred-tasks.md](../deferred-tasks.md#wave-3-step-9b1--recorded-low-findings-2026-10-25): + - a temporary sibling that cannot be removed after a successful link reports `UNEXPECTED`, although the ciphertext + is published; + - no test pins the metafile's 4 MiB value; + - no test builds the container from a real finalizer's staging; + - the overlap check misses containment through a bind mount of a directory inside the other. +- The fifth is corrected in the acceptance commit, which changes no code: the contract and the Windows named limit + no longer call Git for Windows' build the image's only `gpg`. The same commit narrows the contract's overlap + sentence to what the check catches. + +Step 9b.2 is next. From 819f8cbb8d767b4d8803a19c36406493014fd5a4 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 20:32:16 +0300 Subject: [PATCH 77/97] docs(repo): record the Step 9b.2 preflight decisions A read-only preflight at 56ccae62 mapped Step 9b.2's obligations, the code it builds on and what 9c, 9d and Wave 4b need from it. The maintainer approved every recommendation on 2026-10-25: schema v1 carries every contract field now, origin and mapCheck included, checked for shape only; the release-smoke inventory is derived by 9d, not embedded; the toolchain identity is the manifest's per-file inputs plus the tool versions and target; scope is enforced by a new encryptCandidate gate in front of the unchanged encryptOnce; and the production loader has a fixed committed path with an append-only current/retired history, absent in Wave 3. No new ADR or dependency. Also records the NI Windows clause's second green run (run 40211801913 at 56ccae62, probe 0 of 200). Refs: ADR-0111 Co-Authored-By: Claude Opus 5.5 --- .../phase-2.5.5-hardening-and-remediation.md | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index edadf53d..ba8e87fa 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -10057,6 +10057,11 @@ jobs, the advisory Windows job among them. Its probe logged `1 of 200 replacemen negative`: on the runner itself a replaced directory's file ID rounded to its original's once in 200 pairs, the diagnosed mechanism, while every pair stayed distinct as exact identities, and the repeated replacement tests passed. +**Windows clause, run 2 of 3 (2026-10-25).** Push CI +[run 40211801913](https://github.com/HodeTech/Relavium/actions/runs/40211801913) at `56ccae62` passed all eighteen +jobs, the advisory Windows job among them. Its probe logged `0 of 200 replacement pairs collide as Numbers; 0 read +negative`, and the repeated replacement tests passed. + ### Wave 3 Step 9b preflight decisions — 2026-10-25 A read-only preflight (four readers and a synthesis at `c8c0d41b`) mapped Step 9b's obligations, the code it meets and @@ -10333,3 +10338,42 @@ The round's verified findings were five lows: sentence to what the check catches. Step 9b.2 is next. + +### Wave 3 Step 9b.2 preflight decisions — 2026-10-25 + +A read-only preflight (four readers and a synthesis at `56ccae62`) mapped Step 9b.2's obligations, the code it builds +on and what 9c, 9d and Wave 4b will need from it; the detailed record and a Turkish decision register are local and +Git-ignored. The maintainer approved each recommendation on 2026-10-25. No new ADR or runtime dependency follows: the +receipt identifies files by SHA-256 from `node:crypto`, with no signature or HMAC (packaging.md's public binding +receipt), and every module is release tooling. All four parts are new modules, so no accepted 9b.1 file changes and +none of 9b.1's "next change to the file" lows fire. + +| Decision | Ruling | +| --- | --- | +| Schema v1 | Carries every field the contract lists now, `origin` and `mapCheck` included, checked for shape only. 9d adds trust checks against these fields (the origin against the GitHub run, `mapCheck` against a successful `ci` run at the source commit) and no new field; there is no `ciRun` field. | +| Release-smoke inventory | Not embedded. 9d derives the public file inventory and package metadata from the digest-verified tarball with the inspector that produces the package-test receipt, a recorded 9d obligation. | +| Toolchain identity | The manifest's per-file `inputs` (lock and configuration files only), the five tools plus the GnuPG version, and the target: checkable file by file against the commit, and not the per-source inventory ADR-0111 forbids. | +| Scope enforcement | A new `encryptCandidate` gate in front of the unchanged `encryptOnce`, with a module-private registry of the records it returns. Production scope requires the loader's brand and a clean, non-synthetic source; synthetic scope requires the synthetic source, so the per-run ephemeral key can never encrypt a product staging. The sealer accepts only a record that gate returned, and only once. | +| Production loader | A fixed committed path, `tools/cli-package/release-recipient/`, holding `recipient.asc` and a `pins.json` with an append-only `current`/`retired` history: exactly one `current`, no entry ever removed. In Wave 3 the directory is absent and the loader refuses `PRODUCTION_RECIPIENT_ABSENT`; there is no environment variable, flag or fallback, and a fixture fingerprint refuses. Rotation controls are built now (ADR-0111's acceptance 5); Wave 4b only adds the files. | + +Engineering defaults recorded with the rulings: +- **Files.** `release-recipient.mjs` (the loader and its brand), `release-binding.mjs` (schema, validator and reuse, + whose import closure cannot start a process, so reuse's zero GnuPG calls are structural), `release-custody.mjs` (the + gate and the sealer, the only module that reaches GnuPG) and the test-only `release-fixture.mjs`. Each test file + joins `test:acceptance-tools` in the commit that creates it. +- **Receipt bytes.** The receipt is written once in canonical bytes as `release-binding-receipt.json`, the name the + interim guard already keys on. Verifiers parse the exact bytes and never re-serialize, so a duplicated key, a + reordering, a BOM or trailing data refuses. +- **Sealer.** It refuses an existing receipt before reading anything, admits exactly the tarball and the ciphertext + in the destination, and re-reads the ciphertext's size and digest against the record and the tarball's digest + against the manifest. It then publishes once, by `wx` and a link. A sibling that cannot be removed after the link + is not a refusal, and a write port allows fault injection: the lessons of COR-9B1-R4-1 and T-9B1-R1-3. +- **Reuse.** It admits exactly the three files and checks each asset's size, then its bounded bytes and SHA-256, + against the receipt and the expected recipient history. It returns the verified bytes rather than paths and writes + nothing. Re-checking `ENC_TO` would need GnuPG and is a named limit. +- **Static controls.** Only `release-custody.mjs` imports `encryptOnce`; no product path or workflow names a release + module until 9d; the release-binding closure excludes every subprocess module; no recipient directory is tracked + until Wave 4b. The interim guard stays unchanged and still refuses a valid synthetic candidate. +- **Gate.** A rigor pass and a generated mutation sweep before the first round; the full `pnpm run ci` alone; the + custody tests on Ubuntu 24.04 with GnuPG 2.4.4 in Docker; one new complete cumulative round and a separate + crypto/custody security sitting. From 84b8e455cc7f3be18359739ae671ac57167c0f0b Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 21:04:17 +0300 Subject: [PATCH 78/97] feat(repo): add the absent production recipient loader and release scope Step 9b.2's production recipient lives at one committed path, tools/cli-package/release-recipient/, as recipient.asc and an append-only pins history with exactly one current recipient; there is no environment variable, flag or fallback. In Wave 3 the directory does not exist, so the loader refuses PRODUCTION_RECIPIENT_ABSENT; a linked or malformed directory refuses PRODUCTION_RECIPIENT_INVALID and a history naming a committed fixture key FIXTURE_RECIPIENT. A loaded recipient is frozen and branded, so no copy counts as one. The module also names the synthetic candidates' sentinel source. Refs: #46, ADR-0111 Co-Authored-By: Claude Opus 5.5 --- package.json | 2 +- tools/cli-package/release-recipient.mjs | 160 ++++++++++++ tools/cli-package/release-recipient.test.mjs | 260 +++++++++++++++++++ 3 files changed, 421 insertions(+), 1 deletion(-) create mode 100644 tools/cli-package/release-recipient.mjs create mode 100644 tools/cli-package/release-recipient.test.mjs diff --git a/package.json b/package.json index 1f9cee7e..8f7f21f3 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "smoke:budget-replay": "node tools/budget-replay-compat/check.mjs", "check:wire-mutations": "node tools/wire-conformance/check.mjs", "check:usage-mutations": "node tools/usage-coverage-controls/check.mjs", - "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/ustar.test.mjs tools/cli-package/container.test.mjs tools/cli-package/gnupg.test.mjs tools/cli-package/custody.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/check.test.mjs tools/cli-package/paths.test.mjs tools/cli-package/source-map.test.mjs tools/cli-package/manifest.test.mjs tools/cli-package/symbolicate.test.mjs tools/cli-package/map-controls.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", + "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/ustar.test.mjs tools/cli-package/container.test.mjs tools/cli-package/gnupg.test.mjs tools/cli-package/custody.test.mjs tools/cli-package/release-recipient.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/check.test.mjs tools/cli-package/paths.test.mjs tools/cli-package/source-map.test.mjs tools/cli-package/manifest.test.mjs tools/cli-package/symbolicate.test.mjs tools/cli-package/map-controls.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", "check:fs-mutations": "node tools/fs-policy-controls/check.mjs", "check:secrets-mutations": "node tools/secrets-controls/check.mjs", "smoke:cli-bootstrap": "node tools/cli-bootstrap/check.mjs", diff --git a/tools/cli-package/release-recipient.mjs b/tools/cli-package/release-recipient.mjs new file mode 100644 index 00000000..8107f815 --- /dev/null +++ b/tools/cli-package/release-recipient.mjs @@ -0,0 +1,160 @@ +/** + * The production recipient and the release scopes (Step 9b.2; packaging.md, "Recipient custody + * implementation boundary — Step 9b.2"). One committed directory holds the production recipient: + * `recipient.asc`, the `gpg --armor --export '!'` export, and `pins.json`, an append-only history in + * which exactly one recipient is current and none is ever removed, so an earlier candidate still verifies + * after a rotation. There is no environment variable, flag or fallback. In Wave 3 the directory does not + * exist, so every production use refuses; Wave 4b commits the files. A loaded recipient is branded: only the + * value this module returned counts as a production recipient, never a copy of it. + */ +import { lstatSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { isDeepStrictEqual } from 'node:util'; +import { CustodyError, refuseCustody } from './custody-error.mjs'; +import { readOrdinaryFile } from './files.mjs'; + +/** The committed production recipient directory, without a trailing separator so a link is never followed. */ +export const PRODUCTION_RECIPIENT_ROOT = fileURLToPath( + new URL('./release-recipient', import.meta.url), +); +/** The committed synthetic fixtures' pins: no production history may name one of their keys. */ +export const FIXTURE_PINS_FILE = fileURLToPath( + new URL('./custody-fixtures/pins.json', import.meta.url), +); +/** The source every synthetic candidate the fixtures seal records: never a real commit or tree. */ +export const SYNTHETIC_SOURCE = Object.freeze({ commit: 'c'.repeat(40), tree: 'b'.repeat(40) }); + +const FILES = Object.freeze(['pins.json', 'recipient.asc']); +const FILE_BYTES = 64 * 1024; +const MAX_RECIPIENTS = 32; +const FINGERPRINT = /^[0-9A-F]{40}$/u; +/** @type {WeakSet} */ +const loaded = new WeakSet(); + +/** + * @typedef {{primary: string, subkey: string}} Pins + * @typedef {{file: string, current: Pins, history: readonly (Pins & {status: 'current' | 'retired'})[]}} + * ProductionRecipient + */ + +/** @param {unknown} value @returns {value is Record} */ +const isRecord = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); +/** @param {unknown} value */ +const isFingerprint = (value) => typeof value === 'string' && FINGERPRINT.test(value); + +/** Whether `source` is the synthetic candidates' sentinel source. @param {{commit: string, tree: string}} source */ +export const isSyntheticSource = (source) => + source.commit === SYNTHETIC_SOURCE.commit && source.tree === SYNTHETIC_SOURCE.tree; + +/** Whether `value` is a recipient this module loaded: a copy or a hand-built object is not. */ +export const isProductionRecipient = (value) => loaded.has(value); + +/** @param {Buffer} bytes */ +function parseHistory(bytes) { + let value; + try { + value = JSON.parse(bytes.toString('utf8')); + } catch { + return refuseCustody('PRODUCTION_RECIPIENT_INVALID'); + } + if ( + !isRecord(value) || + !isDeepStrictEqual(Object.keys(value).sort(), ['recipients', 'version']) || + value.version !== 1 || + !Array.isArray(value.recipients) || + value.recipients.length < 1 || + value.recipients.length > MAX_RECIPIENTS + ) { + refuseCustody('PRODUCTION_RECIPIENT_INVALID'); + } + const history = value.recipients.map((entry) => { + if ( + !isRecord(entry) || + !isDeepStrictEqual(Object.keys(entry).sort(), ['primary', 'status', 'subkey']) || + !isFingerprint(entry.primary) || + !isFingerprint(entry.subkey) || + (entry.status !== 'current' && entry.status !== 'retired') + ) { + refuseCustody('PRODUCTION_RECIPIENT_INVALID'); + } + return Object.freeze({ primary: entry.primary, subkey: entry.subkey, status: entry.status }); + }); + if ( + history.filter((entry) => entry.status === 'current').length !== 1 || + new Set(history.map((entry) => entry.subkey)).size !== history.length + ) { + refuseCustody('PRODUCTION_RECIPIENT_INVALID'); + } + return Object.freeze(history); +} + +/** Every fingerprint the committed fixtures name. @param {string} file */ +function fixtureFingerprints(file) { + let pins; + try { + pins = JSON.parse(readOrdinaryFile(file, FILE_BYTES).toString('utf8')); + } catch { + return refuseCustody('PRODUCTION_RECIPIENT_INVALID'); + } + if (!isRecord(pins)) refuseCustody('PRODUCTION_RECIPIENT_INVALID'); + return new Set( + Object.values(pins).flatMap((entry) => (isRecord(entry) ? [entry.primary, entry.subkey] : [])), + ); +} + +/** + * Load the production recipient. `root` and `fixtures` are test seams only; no product path passes them. + * @param {{root?: string, fixtures?: string}} [options] + * @returns {ProductionRecipient} + */ +export function loadProductionRecipient({ + root = PRODUCTION_RECIPIENT_ROOT, + fixtures = FIXTURE_PINS_FILE, +} = {}) { + try { + return load(root, fixtures); + } catch (error) { + // A host error's message names a path: only fixed custody codes leave. + if (error instanceof CustodyError) throw error; + return refuseCustody('PRODUCTION_RECIPIENT_INVALID'); + } +} + +/** @param {string} root @param {string} fixtures @returns {ProductionRecipient} */ +function load(root, fixtures) { + let info; + try { + info = lstatSync(root); + } catch (error) { + if (error instanceof Error && 'code' in error && error.code === 'ENOENT') { + refuseCustody('PRODUCTION_RECIPIENT_ABSENT'); + } + throw error; + } + // lstat: a linked directory is not the committed one. + if (!info.isDirectory() || !isDeepStrictEqual(readdirSync(root).sort(), FILES)) { + refuseCustody('PRODUCTION_RECIPIENT_INVALID'); + } + const file = join(root, 'recipient.asc'); + let pins; + try { + readOrdinaryFile(file, FILE_BYTES); + pins = readOrdinaryFile(join(root, 'pins.json'), FILE_BYTES); + } catch { + refuseCustody('PRODUCTION_RECIPIENT_INVALID'); + } + const history = parseHistory(pins); + const forbidden = fixtureFingerprints(fixtures); + if (history.some((entry) => forbidden.has(entry.primary) || forbidden.has(entry.subkey))) { + refuseCustody('FIXTURE_RECIPIENT'); + } + const current = history.find((entry) => entry.status === 'current'); + const recipient = Object.freeze({ + file, + current: Object.freeze({ primary: current.primary, subkey: current.subkey }), + history, + }); + loaded.add(recipient); + return recipient; +} diff --git a/tools/cli-package/release-recipient.test.mjs b/tools/cli-package/release-recipient.test.mjs new file mode 100644 index 00000000..fd5b727f --- /dev/null +++ b/tools/cli-package/release-recipient.test.mjs @@ -0,0 +1,260 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { Buffer } from 'node:buffer'; +import { chmodSync, mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, dirname, join } from 'node:path'; +import { after, test } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { CustodyError } from './custody-error.mjs'; +import { PINS } from './gpg-fixture.mjs'; +import { + FIXTURE_PINS_FILE, + PRODUCTION_RECIPIENT_ROOT, + SYNTHETIC_SOURCE, + isProductionRecipient, + isSyntheticSource, + loadProductionRecipient, +} from './release-recipient.mjs'; + +const scratch = mkdtempSync(join(tmpdir(), 'relavium-release-recipient-')); +after(() => rmSync(scratch, { recursive: true, force: true })); +let count = 0; +const fresh = (name) => join(scratch, `${name}-${(count += 1)}`); +const refuses = (code) => (error) => + error instanceof CustodyError && + error.code === code && + /^PACKED_CLI_CUSTODY_[A-Z_]+$/u.test(error.message) && + !error.message.includes(scratch); +const posix = process.platform !== 'win32'; + +const A = { primary: '1'.repeat(40), subkey: '2'.repeat(40) }; +const B = { primary: '1'.repeat(40), subkey: '3'.repeat(40) }; + +/** A recipient root with the given files; `pins` may be any value or raw text. */ +function root({ + pins = { version: 1, recipients: [{ ...A, status: 'current' }] }, + key = 'key', +} = {}) { + const path = fresh('root'); + mkdirSync(path); + writeFileSync(join(path, 'recipient.asc'), key); + writeFileSync(join(path, 'pins.json'), typeof pins === 'string' ? pins : JSON.stringify(pins)); + return path; +} +const history = (...recipients) => ({ version: 1, recipients }); + +test('the committed recipient directory is absent in Wave 3, so every production use refuses', () => { + // Interim: Wave 4b commits the directory and turns this into a load of the real recipient. + assert.equal(basename(PRODUCTION_RECIPIENT_ROOT), 'release-recipient'); + // No trailing separator: lstat of `dir/` would follow a link to another directory. + assert.equal(/[\\/]$/u.test(PRODUCTION_RECIPIENT_ROOT), false); + assert.equal( + dirname(PRODUCTION_RECIPIENT_ROOT), + fileURLToPath(new URL('.', import.meta.url)).replace(/[\\/]$/u, ''), + ); + assert.throws(() => loadProductionRecipient(), refuses('PRODUCTION_RECIPIENT_ABSENT')); + assert.throws( + () => loadProductionRecipient({ root: join(fresh('missing'), 'release-recipient') }), + refuses('PRODUCTION_RECIPIENT_ABSENT'), + ); +}); + +test('a loaded recipient is frozen, branded, and its current pins come from an append-only history', () => { + const path = root({ pins: history({ ...A, status: 'retired' }, { ...B, status: 'current' }) }); + const recipient = loadProductionRecipient({ root: path }); + assert.equal(recipient.file, join(path, 'recipient.asc')); + assert.deepEqual(recipient.current, B); + assert.deepEqual(recipient.history, [ + { ...A, status: 'retired' }, + { ...B, status: 'current' }, + ]); + assert.ok(Object.isFrozen(recipient) && Object.isFrozen(recipient.current)); + assert.ok(Object.isFrozen(recipient.history) && recipient.history.every(Object.isFrozen)); + assert.equal(isProductionRecipient(recipient), true); + // Only the loaded value is a production recipient: never a copy or a hand-built object. + for (const value of [ + { ...recipient }, + globalThis.structuredClone(recipient), + { file: recipient.file, current: recipient.current, history: recipient.history }, + null, + undefined, + 'recipient', + ]) { + assert.equal(isProductionRecipient(value), false); + } + // Each load brands its own value. + const again = loadProductionRecipient({ root: path }); + assert.notEqual(again, recipient); + assert.equal(isProductionRecipient(again), true); +}); + +test('a recipient directory that is not exactly the two committed files refuses', () => { + const valid = root(); + const cases = [ + () => { + const file = fresh('file-root'); + writeFileSync(file, ''); + return file; + }, + () => { + const path = root(); + writeFileSync(join(path, 'extra'), ''); + return path; + }, + () => { + const path = root(); + rmSync(join(path, 'pins.json')); + return path; + }, + () => { + const path = root(); + rmSync(join(path, 'recipient.asc')); + return path; + }, + () => root({ key: Buffer.alloc(64 * 1024 + 1) }), + () => + root({ + pins: JSON.stringify(history({ ...A, status: 'current' })).padEnd(64 * 1024 + 1, ' '), + }), + ]; + if (posix) { + cases.push( + () => { + const link = fresh('linked-root'); + symlinkSync(valid, link); + return link; + }, + () => { + const path = root(); + rmSync(join(path, 'recipient.asc')); + symlinkSync(join(valid, 'recipient.asc'), join(path, 'recipient.asc')); + return path; + }, + () => { + const path = root(); + rmSync(join(path, 'pins.json')); + assert.equal(spawnSync('mkfifo', [join(path, 'pins.json')]).status, 0); + return path; + }, + ); + } + for (const [index, make] of cases.entries()) { + assert.throws( + () => loadProductionRecipient({ root: make() }), + refuses('PRODUCTION_RECIPIENT_INVALID'), + String(index), + ); + } +}); + +test('the pins history is exactly one current recipient among at most 32 distinct ones', () => { + const current = { ...A, status: 'current' }; + for (const pins of [ + 'not json', + '[]', + history(), + { version: 2, recipients: [current] }, + { version: 1, recipients: [current], extra: true }, + { recipients: [current] }, + { version: 1, recipients: {} }, + history({ ...A, status: 'retired' }), + history(current, { ...B, status: 'current' }), + history(current, { primary: B.primary, subkey: A.subkey, status: 'retired' }), + history({ ...A, status: 'pending' }), + history(current, { ...B, status: 'pending' }), + history({ ...A, status: 'current', extra: true }), + history({ primary: A.primary, status: 'current' }), + history({ primary: 'a'.repeat(40), subkey: A.subkey, status: 'current' }), + history({ primary: [A.primary], subkey: A.subkey, status: 'current' }), + history({ primary: A.primary, subkey: `${A.subkey}!`, status: 'current' }), + history( + current, + ...Array.from({ length: 32 }, (_, index) => ({ + primary: A.primary, + subkey: index.toString(16).toUpperCase().padStart(40, 'F'), + status: 'retired', + })), + ), + ]) { + assert.throws( + () => loadProductionRecipient({ root: root({ pins }) }), + refuses('PRODUCTION_RECIPIENT_INVALID'), + typeof pins === 'string' ? pins : JSON.stringify(pins).slice(0, 120), + ); + } + // Exactly 32 entries is within the bound. + const full = history( + current, + ...Array.from({ length: 31 }, (_, index) => ({ + primary: A.primary, + subkey: index.toString(16).toUpperCase().padStart(40, 'F'), + status: 'retired', + })), + ); + assert.equal(loadProductionRecipient({ root: root({ pins: full }) }).history.length, 32); +}); + +test('a history naming a committed fixture key refuses, whichever fingerprint it is', () => { + const fixture = PINS.good; + for (const entry of [ + { primary: fixture.primary, subkey: A.subkey }, + { primary: A.primary, subkey: fixture.subkey }, + ]) { + assert.throws( + () => + loadProductionRecipient({ + root: root({ pins: history({ ...entry, status: 'current' }) }), + }), + refuses('FIXTURE_RECIPIENT'), + ); + } + // A retired fixture entry refuses as well. + assert.throws( + () => + loadProductionRecipient({ + root: root({ + pins: history({ ...A, status: 'current' }, { ...PINS.other, status: 'retired' }), + }), + }), + refuses('FIXTURE_RECIPIENT'), + ); + // The denylist is the committed fixtures file; an unreadable or malformed one fails closed. + assert.equal( + FIXTURE_PINS_FILE, + fileURLToPath(new URL('./custody-fixtures/pins.json', import.meta.url)), + ); + const malformed = fresh('fixtures.json'); + writeFileSync(malformed, '[]'); + for (const fixtures of [join(fresh('none'), 'pins.json'), malformed]) { + assert.throws( + () => loadProductionRecipient({ root: root(), fixtures }), + refuses('PRODUCTION_RECIPIENT_INVALID'), + ); + } +}); + +test( + 'a recipient directory that cannot be read refuses with a fixed code and no path', + { skip: !posix || process.getuid?.() === 0 }, + () => { + const path = root(); + chmodSync(path, 0o000); + try { + assert.throws( + () => loadProductionRecipient({ root: path }), + refuses('PRODUCTION_RECIPIENT_INVALID'), + ); + } finally { + chmodSync(path, 0o700); + } + }, +); + +test('the synthetic source is the fixtures sentinel, never a clean source', () => { + assert.deepEqual(SYNTHETIC_SOURCE, { commit: 'c'.repeat(40), tree: 'b'.repeat(40) }); + assert.ok(Object.isFrozen(SYNTHETIC_SOURCE)); + assert.equal(isSyntheticSource({ commit: 'c'.repeat(40), tree: 'b'.repeat(40) }), true); + assert.equal(isSyntheticSource({ commit: 'c'.repeat(40), tree: 'e'.repeat(40) }), false); + assert.equal(isSyntheticSource({ commit: 'd'.repeat(40), tree: 'b'.repeat(40) }), false); +}); From cbf6ea45d1ca4d362a6e82ef646056c0cb1a9d9e Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 21:04:17 +0300 Subject: [PATCH 79/97] feat(repo): add the release binding receipt v1, its validator and verified reuse The receipt carries every field the contract lists, origin and mapCheck included, checked for shape until Step 9d binds them to the trusted run. It is written once in canonical bytes, and a verifier refuses anything but the canonical serialization of the value it validates. The expectation is required: production needs the loaded recipient, whose whole history verifies, and a clean non-synthetic source; synthetic needs the sentinel source and its pins. verifyRetainedCandidate admits exactly the receipt, tarball and ciphertext, checks each asset's size and SHA-256 against the receipt and returns the verified bytes. The module's import closure starts no process, so reuse makes no GnuPG call. Refs: #46, ADR-0111 Co-Authored-By: Claude Opus 5.5 --- package.json | 2 +- tools/cli-package/release-binding.mjs | 328 ++++++++++ tools/cli-package/release-binding.test.mjs | 692 +++++++++++++++++++++ 3 files changed, 1021 insertions(+), 1 deletion(-) create mode 100644 tools/cli-package/release-binding.mjs create mode 100644 tools/cli-package/release-binding.test.mjs diff --git a/package.json b/package.json index 8f7f21f3..960bf20a 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "smoke:budget-replay": "node tools/budget-replay-compat/check.mjs", "check:wire-mutations": "node tools/wire-conformance/check.mjs", "check:usage-mutations": "node tools/usage-coverage-controls/check.mjs", - "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/ustar.test.mjs tools/cli-package/container.test.mjs tools/cli-package/gnupg.test.mjs tools/cli-package/custody.test.mjs tools/cli-package/release-recipient.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/check.test.mjs tools/cli-package/paths.test.mjs tools/cli-package/source-map.test.mjs tools/cli-package/manifest.test.mjs tools/cli-package/symbolicate.test.mjs tools/cli-package/map-controls.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", + "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/ustar.test.mjs tools/cli-package/container.test.mjs tools/cli-package/gnupg.test.mjs tools/cli-package/custody.test.mjs tools/cli-package/release-recipient.test.mjs tools/cli-package/release-binding.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/check.test.mjs tools/cli-package/paths.test.mjs tools/cli-package/source-map.test.mjs tools/cli-package/manifest.test.mjs tools/cli-package/symbolicate.test.mjs tools/cli-package/map-controls.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", "check:fs-mutations": "node tools/fs-policy-controls/check.mjs", "check:secrets-mutations": "node tools/secrets-controls/check.mjs", "smoke:cli-bootstrap": "node tools/cli-bootstrap/check.mjs", diff --git a/tools/cli-package/release-binding.mjs b/tools/cli-package/release-binding.mjs new file mode 100644 index 00000000..aac27d17 --- /dev/null +++ b/tools/cli-package/release-binding.mjs @@ -0,0 +1,328 @@ +/** + * The release binding receipt (Step 9b.2; packaging.md, "Recipient custody implementation boundary — + * Step 9b.2"): schema v1, its validator and the verified reuse of a retained candidate. The receipt is + * public JSON beside the ciphertext, written once in canonical bytes. It names files by SHA-256 and signs + * nothing: every field is a claim, and its origin, map-check and `ci` fields are checked for shape only + * until Step 9d binds them to the trusted run. Scope always comes from the verifier's expectation, never + * from the receipt. This module's import closure starts no process, so reuse makes no GnuPG call. + */ +import { Buffer } from 'node:buffer'; +import { lstatSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { isDeepStrictEqual, TextDecoder } from 'node:util'; +import { ARCHIVE_LIMITS } from './archive.mjs'; +import { CustodyError, refuseCustody } from './custody-error.mjs'; +import { readOrdinaryFile, sha256 } from './files.mjs'; +import { isProductionRecipient, isSyntheticSource } from './release-recipient.mjs'; + +/** The receipt's file name beside the ciphertext, as the interim release guard keys on it. */ +export const RELEASE_BINDING_FILE = 'release-binding-receipt.json'; +export const RELEASE_BINDING_KIND = 'release_binding'; +/** The custody container's 128 MiB total plus the 64 KiB a ciphertext may add (container.mjs, custody.mjs). */ +export const CIPHERTEXT_BYTES = 128 * 1024 * 1024 + 64 * 1024; +const RECEIPT_BYTES = 64 * 1024; +const MAX_INPUTS = 64; + +/** @param {string} version */ +export const tarballName = (version) => `relavium-${version}.tgz`; +/** @param {string} version */ +export const ciphertextFileName = (version) => `relavium-${version}-private-maps.tar.gpg`; + +const KEYS = Object.freeze([ + 'version', + 'kind', + 'packageVersion', + 'source', + 'tarball', + 'ciphertext', + 'recipient', + 'toolchain', + 'target', + 'inputs', + 'mapCheck', + 'origin', +]); +const SHA256 = /^[0-9a-f]{64}$/u; +const GIT_OBJECT = /^[0-9a-f]{40}$/u; +const FINGERPRINT = /^[0-9A-F]{40}$/u; +const PACKAGE_VERSION = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/u; +const TOOL_VERSION = /^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]{1,64})?$/u; +const SEGMENT = /^[A-Za-z0-9_.-]+$/u; + +/** + * @typedef {import('./release-recipient.mjs').ProductionRecipient} ProductionRecipient + * @typedef {{scope: 'production', recipient: ProductionRecipient, packageVersion: string, source: string} + * | {scope: 'synthetic', pins: {primary: string, subkey: string}, packageVersion: string}} Expected + */ + +/** @param {unknown} value @returns {value is Record} */ +const isRecord = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); + +/** @param {unknown} value @param {readonly string[]} keys */ +function exact(value, keys) { + if (!isRecord(value) || !isDeepStrictEqual(Object.keys(value).sort(), [...keys].sort())) { + refuseCustody('BINDING_SHAPE'); + } + return /** @type {Record} */ (value); +} +/** @param {unknown} value @param {RegExp} pattern @param {number} [max] @returns {string} */ +function string(value, pattern, max = 256) { + if (typeof value !== 'string' || value.length > max || !pattern.test(value)) { + refuseCustody('BINDING_SHAPE'); + } + return /** @type {string} */ (value); +} +/** @param {unknown} value @param {number} minimum @param {number} maximum @returns {number} */ +function size(value, minimum, maximum) { + if (typeof value !== 'number' || !Number.isSafeInteger(value)) refuseCustody('BINDING_SHAPE'); + if (value < minimum || value > maximum) refuseCustody('BINDING_SIZE'); + return value; +} +/** A repository-relative POSIX path with no empty, `.` or `..` segment. @param {string} path */ +const isRelativePath = (path) => + path.length <= 256 && + path.split('/').every((segment) => SEGMENT.test(segment) && segment !== '.' && segment !== '..'); + +/** @param {unknown} expected */ +function scopeOf(expected) { + if (!isRecord(expected)) return refuseCustody('BINDING_SCOPE'); + if (expected.scope === 'production') { + if (!isProductionRecipient(expected.recipient)) refuseCustody('RECIPIENT_SOURCE'); + } else if (expected.scope !== 'synthetic') { + refuseCustody('BINDING_SCOPE'); + } + return /** @type {Expected} */ (expected); +} + +/** + * @param {unknown} value @param {string} version @param {(version: string) => string} name + * @param {number} minimum @param {number} maximum + */ +function asset(value, version, name, minimum, maximum) { + const entry = exact(value, ['name', 'bytes', 'sha256']); + if (entry.name !== name(version)) refuseCustody('BINDING_NAME'); + return Object.freeze({ + name: /** @type {string} */ (entry.name), + bytes: size(entry.bytes, minimum, maximum), + sha256: string(entry.sha256, SHA256), + }); +} + +/** + * Validate a release binding receipt against the verifier's expectation and return its normalized, + * frozen copy in canonical key order. `expected` has no default: a production expectation needs the + * loaded production recipient, whose whole history verifies; a synthetic one needs the synthetic pins. + * @param {unknown} value + * @param {Expected} expected + */ +export function validateReleaseBinding(value, expected) { + const scope = scopeOf(expected); + if (!isRecord(value) || value.version !== 1 || value.kind !== RELEASE_BINDING_KIND) { + refuseCustody('BINDING_VERSION'); + } + const receipt = exact(value, KEYS); + const packageVersion = string(receipt.packageVersion, PACKAGE_VERSION, 100); + const sourceRecord = exact(receipt.source, ['commit', 'tree', 'dirty']); + if (typeof sourceRecord.dirty !== 'boolean') refuseCustody('BINDING_SHAPE'); + const source = Object.freeze({ + commit: string(sourceRecord.commit, GIT_OBJECT), + tree: string(sourceRecord.tree, GIT_OBJECT), + dirty: sourceRecord.dirty, + }); + if (packageVersion !== scope.packageVersion) refuseCustody('BINDING_CANDIDATE'); + if (scope.scope === 'production') { + if (source.commit !== scope.source) refuseCustody('BINDING_CANDIDATE'); + if (isSyntheticSource(source)) refuseCustody('BINDING_SCOPE'); + if (source.dirty) refuseCustody('BINDING_DIRTY'); + } else if (!isSyntheticSource(source)) { + refuseCustody('BINDING_SCOPE'); + } + + // A gzip member is at least 18 bytes (archive.mjs); a ciphertext at least one. + const tarball = asset( + receipt.tarball, + packageVersion, + tarballName, + 18, + ARCHIVE_LIMITS.compressedBytes, + ); + const ciphertext = asset( + receipt.ciphertext, + packageVersion, + ciphertextFileName, + 1, + CIPHERTEXT_BYTES, + ); + + const recipientRecord = exact(receipt.recipient, ['primary', 'subkey']); + const recipient = Object.freeze({ + primary: string(recipientRecord.primary, FINGERPRINT), + subkey: string(recipientRecord.subkey, FINGERPRINT), + }); + const admitted = + scope.scope === 'production' + ? scope.recipient.history + : [{ primary: scope.pins?.primary, subkey: scope.pins?.subkey }]; + if ( + !admitted.some( + (entry) => entry.primary === recipient.primary && entry.subkey === recipient.subkey, + ) + ) { + refuseCustody('BINDING_RECIPIENT'); + } + + const tools = exact(receipt.toolchain, [ + 'node', + 'pnpm', + 'typescript', + 'tsup', + 'esbuild', + 'gnupg', + ]); + const toolchain = Object.freeze({ + node: string(tools.node, /^v\d+\.\d+\.\d+$/u, 40), + pnpm: string(tools.pnpm, TOOL_VERSION, 100), + typescript: string(tools.typescript, TOOL_VERSION, 100), + tsup: string(tools.tsup, TOOL_VERSION, 100), + esbuild: string(tools.esbuild, TOOL_VERSION, 100), + gnupg: string(tools.gnupg, /^\d+\.\d+\.\d+$/u, 40), + }); + const target = string(receipt.target, /^node\d{2,3}$/u, 8); + + if (!isRecord(receipt.inputs)) refuseCustody('BINDING_SHAPE'); + const entries = Object.entries(receipt.inputs); + if (entries.length < 1 || entries.length > MAX_INPUTS) refuseCustody('BINDING_SHAPE'); + const inputs = Object.freeze( + Object.fromEntries( + entries + .map(([path, digest]) => { + if (!isRelativePath(path)) refuseCustody('BINDING_SHAPE'); + return [path, string(digest, SHA256)]; + }) + .sort(([left], [right]) => (left < right ? -1 : 1)), + ), + ); + + const mapCheck = exact(receipt.mapCheck, ['status']); + if (mapCheck.status !== 'complete') refuseCustody('BINDING_MAP_CHECK'); + + if (!isRecord(receipt.origin) || receipt.origin.kind !== 'github') { + refuseCustody('BINDING_ORIGIN'); + } + const originRecord = exact(receipt.origin, [ + 'kind', + 'repository', + 'workflow', + 'runId', + 'runAttempt', + 'event', + 'source', + ]); + if (originRecord.event !== 'push' || originRecord.source !== source.commit) { + refuseCustody('BINDING_ORIGIN'); + } + const origin = Object.freeze({ + kind: 'github', + repository: string(originRecord.repository, /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u, 200), + workflow: string(originRecord.workflow, /^\P{Cc}+$/u, 200), + runId: string(originRecord.runId, /^[1-9]\d*$/u, 20), + runAttempt: string(originRecord.runAttempt, /^[1-9]\d*$/u, 6), + event: 'push', + source: source.commit, + }); + + return Object.freeze({ + version: 1, + kind: RELEASE_BINDING_KIND, + packageVersion, + source, + tarball, + ciphertext, + recipient, + toolchain, + target, + inputs, + mapCheck: Object.freeze({ status: 'complete' }), + origin, + }); +} + +/** The canonical bytes of a normalized receipt; the sealer writes exactly these, once. @param {object} receipt */ +export const serializeReleaseBinding = (receipt) => + Buffer.from(`${JSON.stringify(receipt, null, 2)}\n`); + +/** + * Parse a receipt's exact bytes: anything but the canonical serialization of its validated value + * refuses, so a duplicated key, a reordering, a BOM or trailing data never reaches a verifier. + * @param {Buffer} bytes @param {Expected} expected + */ +export function parseReleaseBinding(bytes, expected) { + if (!Buffer.isBuffer(bytes) || bytes.length > RECEIPT_BYTES) refuseCustody('BINDING_BYTES'); + let value; + try { + value = JSON.parse(new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(bytes)); + } catch { + return refuseCustody('BINDING_BYTES'); + } + const receipt = validateReleaseBinding(value, expected); + if (!serializeReleaseBinding(receipt).equals(bytes)) refuseCustody('BINDING_BYTES'); + return receipt; +} + +/** + * Verify a retained candidate for reuse: the directory holds exactly its receipt, tarball and ciphertext, + * the receipt is valid for the expectation, and each asset has the receipt's size and SHA-256. Returns the + * verified bytes, never a path to open again; writes, renames and removes nothing. + * @param {{directory: string, expected: Expected}} options + */ +export function verifyRetainedCandidate({ directory, expected }) { + try { + return verify(directory, expected); + } catch (error) { + // A host error's message names a path: only fixed custody codes leave. + if (error instanceof CustodyError) throw error; + return refuseCustody('UNEXPECTED'); + } +} + +/** @param {string} directory @param {Expected} expected */ +function verify(directory, expected) { + const { packageVersion } = scopeOf(expected); + if (typeof packageVersion !== 'string' || !PACKAGE_VERSION.test(packageVersion)) { + refuseCustody('BINDING_CANDIDATE'); + } + const names = [ + RELEASE_BINDING_FILE, + tarballName(packageVersion), + ciphertextFileName(packageVersion), + ].sort(); + if (!isDeepStrictEqual(readdirSync(directory).sort(), names)) refuseCustody('CANDIDATE_ENTRIES'); + let bytes; + try { + bytes = readOrdinaryFile(join(directory, RELEASE_BINDING_FILE), RECEIPT_BYTES); + } catch { + refuseCustody('CANDIDATE_ASSET'); + } + const receipt = parseReleaseBinding(bytes, expected); + /** @param {{name: string, bytes: number, sha256: string}} entry */ + const retained = (entry) => { + const path = join(directory, entry.name); + const info = lstatSync(path); + if (!info.isFile()) refuseCustody('CANDIDATE_ASSET'); + if (info.size !== entry.bytes) refuseCustody('CANDIDATE_SIZE'); + let data; + try { + data = readOrdinaryFile(path, entry.bytes); + } catch { + return refuseCustody('CANDIDATE_ASSET'); + } + if (sha256(data) !== entry.sha256) refuseCustody('CANDIDATE_DIGEST'); + return data; + }; + return Object.freeze({ + receipt, + receiptBytes: bytes, + tarball: retained(receipt.tarball), + ciphertext: retained(receipt.ciphertext), + }); +} diff --git a/tools/cli-package/release-binding.test.mjs b/tools/cli-package/release-binding.test.mjs new file mode 100644 index 00000000..ea1a78d6 --- /dev/null +++ b/tools/cli-package/release-binding.test.mjs @@ -0,0 +1,692 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { Buffer } from 'node:buffer'; +import { + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + readlinkSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { after, test } from 'node:test'; +import { ARCHIVE_LIMITS } from './archive.mjs'; +import { isPackageVersion } from './binding.mjs'; +import { CustodyError } from './custody-error.mjs'; +import { parsePins } from './custody.mjs'; +import { sha256 } from './files.mjs'; +import { + CIPHERTEXT_BYTES, + RELEASE_BINDING_FILE, + RELEASE_BINDING_KIND, + ciphertextFileName, + parseReleaseBinding, + serializeReleaseBinding, + tarballName, + validateReleaseBinding, + verifyRetainedCandidate, +} from './release-binding.mjs'; +import { SYNTHETIC_SOURCE, loadProductionRecipient } from './release-recipient.mjs'; + +const scratch = mkdtempSync(join(tmpdir(), 'relavium-release-binding-')); +after(() => rmSync(scratch, { recursive: true, force: true })); +let count = 0; +const fresh = (name) => join(scratch, `${name}-${(count += 1)}`); +const refuses = (code) => (error) => + error instanceof CustodyError && + error.code === code && + /^PACKED_CLI_CUSTODY_[A-Z_]+$/u.test(error.message) && + !error.message.includes(scratch); +const posix = process.platform !== 'win32'; + +const VERSION = '0.1.1'; +const TARBALL = Buffer.from('a synthetic tarball, never a package\n'); +const CIPHERTEXT = Buffer.from('synthetic ciphertext bytes'); +const PINS = Object.freeze({ primary: '4'.repeat(40), subkey: '5'.repeat(40) }); +const A = { primary: '1'.repeat(40), subkey: '2'.repeat(40) }; +const B = { primary: '1'.repeat(40), subkey: '3'.repeat(40) }; +const C = { primary: '6'.repeat(40), subkey: '7'.repeat(40) }; +const CLEAN = { commit: 'd'.repeat(40), tree: 'e'.repeat(40), dirty: false }; +const SYNTHETIC = { ...SYNTHETIC_SOURCE, dirty: true }; + +/** A loaded production recipient with the given history, from a temporary root. */ +function production(...recipients) { + const root = fresh('recipient'); + mkdirSync(root); + writeFileSync(join(root, 'recipient.asc'), 'key'); + writeFileSync(join(root, 'pins.json'), JSON.stringify({ version: 1, recipients })); + return loadProductionRecipient({ root }); +} +const ROTATED = production({ ...A, status: 'retired' }, { ...B, status: 'current' }); +const synthetic = Object.freeze({ scope: 'synthetic', pins: PINS, packageVersion: VERSION }); +const productionExpected = (recipient = ROTATED) => + Object.freeze({ + scope: 'production', + recipient, + packageVersion: VERSION, + source: CLEAN.commit, + }); + +/** A receipt in canonical key order; synthetic unless told otherwise. */ +function receipt({ + source = SYNTHETIC, + recipient = PINS, + tarball = TARBALL, + ciphertext = CIPHERTEXT, +} = {}) { + return { + version: 1, + kind: RELEASE_BINDING_KIND, + packageVersion: VERSION, + source: { ...source }, + tarball: { name: tarballName(VERSION), bytes: tarball.length, sha256: sha256(tarball) }, + ciphertext: { + name: ciphertextFileName(VERSION), + bytes: ciphertext.length, + sha256: sha256(ciphertext), + }, + recipient: { ...recipient }, + toolchain: { + node: 'v22.23.1', + pnpm: '9.12.3', + typescript: '5.6.3', + tsup: '8.3.5', + esbuild: '0.24.0', + gnupg: '2.5.21', + }, + target: 'node22', + inputs: { 'apps/cli/package.json': 'a'.repeat(64), 'package.json': 'b'.repeat(64) }, + mapCheck: { status: 'complete' }, + origin: { + kind: 'github', + repository: 'example/relavium', + workflow: 'release', + runId: '1', + runAttempt: '1', + event: 'push', + source: source.commit, + }, + }; +} +const productionReceipt = (recipient = B) => receipt({ source: CLEAN, recipient }); +/** A deep copy changed by `change`. */ +const changed = (value, change) => { + const copy = globalThis.structuredClone(value); + change(copy); + return copy; +}; + +test('a valid receipt validates to its frozen canonical copy, and its canonical bytes parse', () => { + for (const [value, expected] of [ + [receipt(), synthetic], + [productionReceipt(), productionExpected()], + ]) { + const normalized = validateReleaseBinding(value, expected); + assert.deepEqual(normalized, value); + assert.notEqual(normalized, value); + assert.deepEqual(Object.keys(normalized), Object.keys(value)); + const frozen = (object) => + Object.isFrozen(object) && + Object.values(object).every((child) => typeof child !== 'object' || frozen(child)); + assert.ok(frozen(normalized)); + const bytes = serializeReleaseBinding(normalized); + assert.equal(bytes.toString('utf8'), `${JSON.stringify(value, null, 2)}\n`); + assert.deepEqual(parseReleaseBinding(bytes, expected), normalized); + } + // Inputs come back sorted by path. + const unsorted = changed(receipt(), (copy) => { + copy.inputs = { 'package.json': 'b'.repeat(64), 'apps/cli/package.json': 'a'.repeat(64) }; + }); + assert.deepEqual(Object.keys(validateReleaseBinding(unsorted, synthetic).inputs), [ + 'apps/cli/package.json', + 'package.json', + ]); +}); + +test('every key at every level is exact', () => { + const value = receipt(); + for (const key of Object.keys(value)) { + const code = key === 'version' || key === 'kind' ? 'BINDING_VERSION' : 'BINDING_SHAPE'; + assert.throws( + () => + validateReleaseBinding( + changed(value, (copy) => delete copy[key]), + synthetic, + ), + refuses(code), + key, + ); + } + assert.throws( + () => validateReleaseBinding({ ...value, extra: true }, synthetic), + refuses('BINDING_SHAPE'), + ); + for (const parent of [ + 'source', + 'tarball', + 'ciphertext', + 'recipient', + 'toolchain', + 'mapCheck', + 'origin', + ]) { + for (const key of Object.keys(value[parent])) { + const code = + parent === 'origin' && key === 'kind' + ? 'BINDING_ORIGIN' + : parent === 'origin' && key === 'source' + ? 'BINDING_SHAPE' + : 'BINDING_SHAPE'; + assert.throws( + () => + validateReleaseBinding( + changed(value, (copy) => delete copy[parent][key]), + synthetic, + ), + refuses(code), + `${parent}.${key}`, + ); + } + assert.throws( + () => + validateReleaseBinding( + changed(value, (copy) => (copy[parent].extra = 'x')), + synthetic, + ), + refuses('BINDING_SHAPE'), + `${parent}.extra`, + ); + for (const replacement of [null, [], 'x']) { + const code = parent === 'origin' ? 'BINDING_ORIGIN' : 'BINDING_SHAPE'; + assert.throws( + () => + validateReleaseBinding( + changed(value, (copy) => (copy[parent] = replacement)), + synthetic, + ), + refuses(code), + `${parent} = ${JSON.stringify(replacement)}`, + ); + } + } + for (const inputs of [null, [], 'x']) { + assert.throws( + () => + validateReleaseBinding( + changed(value, (copy) => (copy.inputs = inputs)), + synthetic, + ), + refuses('BINDING_SHAPE'), + ); + } +}); + +test('every value has its exact format', () => { + const value = receipt(); + const cases = [ + (copy) => (copy.packageVersion = 'v0.1.1'), + (copy) => (copy.packageVersion = `0.1.1-${'a'.repeat(100)}`), + (copy) => (copy.source.commit = copy.source.commit.toUpperCase()), + (copy) => (copy.source.tree = 'b'.repeat(39)), + (copy) => (copy.source.dirty = 'true'), + (copy) => (copy.tarball.sha256 = copy.tarball.sha256.toUpperCase()), + (copy) => (copy.ciphertext.sha256 = 'z'.repeat(64)), + (copy) => (copy.tarball.bytes = 18.5), + (copy) => (copy.ciphertext.bytes = '26'), + (copy) => (copy.recipient.primary = copy.recipient.primary.toLowerCase().replace(/4/gu, 'a')), + (copy) => (copy.recipient.subkey = `${copy.recipient.subkey.slice(0, 39)}!`), + (copy) => (copy.toolchain.node = '22.23.1'), + (copy) => (copy.toolchain.pnpm = 'latest'), + (copy) => (copy.toolchain.typescript = '5.6'), + (copy) => (copy.toolchain.tsup = 8), + (copy) => (copy.toolchain.esbuild = '0.24.0-'), + (copy) => (copy.toolchain.gnupg = '2.5.21-beta'), + (copy) => (copy.target = 'node2'), + (copy) => (copy.target = 'deno22'), + (copy) => (copy.inputs = {}), + (copy) => + (copy.inputs = Object.fromEntries( + Array.from({ length: 65 }, (_, index) => [`file-${index}.json`, 'a'.repeat(64)]), + )), + ...['../package.json', './package.json', 'apps//cli', '/package.json', 'apps\\cli', 'a b'].map( + (path) => (copy) => (copy.inputs = { [path]: 'a'.repeat(64) }), + ), + (copy) => (copy.inputs = { [`${'a'.repeat(257)}`]: 'a'.repeat(64) }), + (copy) => (copy.inputs = { 'package.json': 'A'.repeat(64) }), + (copy) => (copy.origin.repository = 'relavium'), + (copy) => (copy.origin.workflow = 'release\u0007'), + (copy) => (copy.origin.runId = '0'), + (copy) => (copy.origin.runAttempt = '01'), + (copy) => (copy.origin.runAttempt = '1234567'), + ]; + for (const [index, change] of cases.entries()) { + assert.throws( + () => validateReleaseBinding(changed(value, change), synthetic), + refuses('BINDING_SHAPE'), + `case ${index}`, + ); + } + // At the bounds: 64 inputs and a 256-character path are admitted. + const bounded = changed(value, (copy) => { + copy.inputs = Object.fromEntries( + Array.from({ length: 64 }, (_, index) => [ + index === 0 ? 'a'.repeat(256) : `file-${String(index).padStart(2, '0')}.json`, + 'a'.repeat(64), + ]), + ); + }); + assert.equal(Object.keys(validateReleaseBinding(bounded, synthetic).inputs).length, 64); + const single = changed(value, (copy) => (copy.inputs = { 'package.json': 'a'.repeat(64) })); + assert.deepEqual(validateReleaseBinding(single, synthetic).inputs, { + 'package.json': 'a'.repeat(64), + }); +}); + +test('the version, candidate, names, sizes, map check and origin refuse with their own codes', () => { + const value = receipt(); + for (const [code, change] of [ + ['BINDING_VERSION', (copy) => (copy.version = 2)], + ['BINDING_VERSION', (copy) => (copy.kind = 'package_test')], + ['BINDING_CANDIDATE', (copy) => (copy.packageVersion = '0.1.2')], + ['BINDING_NAME', (copy) => (copy.tarball.name = 'relavium-0.1.2.tgz')], + ['BINDING_NAME', (copy) => (copy.ciphertext.name = 'relavium-0.1.1.tgz.gpg')], + ['BINDING_SIZE', (copy) => (copy.tarball.bytes = 17)], + ['BINDING_SIZE', (copy) => (copy.tarball.bytes = ARCHIVE_LIMITS.compressedBytes + 1)], + ['BINDING_SIZE', (copy) => (copy.ciphertext.bytes = 0)], + ['BINDING_SIZE', (copy) => (copy.ciphertext.bytes = CIPHERTEXT_BYTES + 1)], + ['BINDING_MAP_CHECK', (copy) => (copy.mapCheck.status = 'pending')], + ['BINDING_ORIGIN', (copy) => (copy.origin = { kind: 'local' })], + ['BINDING_ORIGIN', (copy) => (copy.origin.event = 'pull_request')], + ['BINDING_ORIGIN', (copy) => (copy.origin.event = 'workflow_dispatch')], + ['BINDING_ORIGIN', (copy) => (copy.origin.source = 'f'.repeat(40))], + ['BINDING_RECIPIENT', (copy) => (copy.recipient = { ...C })], + ['BINDING_RECIPIENT', (copy) => (copy.recipient.subkey = C.subkey)], + ['BINDING_SCOPE', (copy) => (copy.source = { ...CLEAN })], + ['BINDING_SCOPE', (copy) => (copy.source.tree = 'e'.repeat(40))], + ]) { + assert.throws( + () => validateReleaseBinding(changed(value, change), synthetic), + refuses(code), + code, + ); + } + for (const bad of [null, [], 'receipt', 1]) { + assert.throws(() => validateReleaseBinding(bad, synthetic), refuses('BINDING_VERSION')); + } + // The sizes are inclusive bounds. + for (const change of [ + (copy) => (copy.tarball.bytes = 18), + (copy) => (copy.tarball.bytes = ARCHIVE_LIMITS.compressedBytes), + (copy) => (copy.ciphertext.bytes = 1), + (copy) => (copy.ciphertext.bytes = CIPHERTEXT_BYTES), + ]) { + validateReleaseBinding(changed(value, change), synthetic); + } + assert.equal(CIPHERTEXT_BYTES, 128 * 1024 * 1024 + 64 * 1024); +}); + +test('production scope admits only a clean, non-synthetic source of the expected commit and a recipient from the history', () => { + const value = productionReceipt(); + for (const [code, change] of [ + ['BINDING_CANDIDATE', (copy) => (copy.source.commit = 'f'.repeat(40))], + ['BINDING_DIRTY', (copy) => (copy.source.dirty = true)], + ['BINDING_RECIPIENT', (copy) => (copy.recipient = { ...C })], + ['BINDING_RECIPIENT', (copy) => (copy.recipient = { primary: A.primary, subkey: C.subkey })], + ]) { + assert.throws( + () => + validateReleaseBinding( + changed(value, (copy) => { + change(copy); + copy.origin.source = copy.source.commit; + }), + productionExpected(), + ), + refuses(code), + code, + ); + } + // The synthetic sentinel never validates as a release, even with the expected commit. + const sentinel = receipt({ source: { ...SYNTHETIC_SOURCE, dirty: false }, recipient: B }); + assert.throws( + () => + validateReleaseBinding(sentinel, { + ...productionExpected(), + source: SYNTHETIC_SOURCE.commit, + }), + refuses('BINDING_SCOPE'), + ); + // A rotation keeps every earlier candidate valid: a retired recipient still verifies, a recipient + // outside the history never does, and a history that dropped one refuses that one's candidates. + validateReleaseBinding(productionReceipt(A), productionExpected()); + validateReleaseBinding(productionReceipt(B), productionExpected()); + assert.throws( + () => validateReleaseBinding(productionReceipt(C), productionExpected()), + refuses('BINDING_RECIPIENT'), + ); + const dropped = production({ ...B, status: 'current' }); + assert.throws( + () => validateReleaseBinding(productionReceipt(A), productionExpected(dropped)), + refuses('BINDING_RECIPIENT'), + ); +}); + +test('the expectation is required, and production needs the loaded recipient itself', () => { + const value = productionReceipt(); + for (const expected of [ + undefined, + null, + {}, + { scope: 'release' }, + { ...synthetic, scope: 'Synthetic' }, + ]) { + assert.throws(() => validateReleaseBinding(value, expected), refuses('BINDING_SCOPE')); + } + for (const recipient of [ + { ...ROTATED }, + globalThis.structuredClone(ROTATED), + { file: ROTATED.file, current: ROTATED.current, history: ROTATED.history }, + undefined, + ]) { + assert.throws( + () => validateReleaseBinding(value, { ...productionExpected(), recipient }), + refuses('RECIPIENT_SOURCE'), + ); + } + // A synthetic expectation without pins admits no recipient. + assert.throws( + () => validateReleaseBinding(receipt(), { scope: 'synthetic', packageVersion: VERSION }), + refuses('BINDING_RECIPIENT'), + ); +}); + +test('only the exact canonical bytes parse', () => { + const bytes = serializeReleaseBinding(validateReleaseBinding(receipt(), synthetic)); + const text = bytes.toString('utf8'); + for (const [name, bad] of [ + ['duplicate key', Buffer.from(text.replace('{\n', '{\n "version": 1,\n'))], + [ + 'reordered keys', + Buffer.from(`${JSON.stringify({ kind: RELEASE_BINDING_KIND, ...receipt() }, null, 2)}\n`), + ], + ['compact', Buffer.from(`${JSON.stringify(receipt())}\n`)], + ['no final newline', Buffer.from(text.slice(0, -1))], + ['BOM', Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), bytes])], + ['trailing byte', Buffer.concat([bytes, Buffer.from(' ')])], + ['CRLF', Buffer.from(text.replaceAll('\n', '\r\n'))], + [ + 'invalid UTF-8', + Buffer.concat([bytes.subarray(0, -2), Buffer.from([0xff]), Buffer.from('\n')]), + ], + ['not JSON', Buffer.from('receipt')], + ['too large', Buffer.concat([bytes, Buffer.alloc(64 * 1024 + 1 - bytes.length, 0x20)])], + [ + 'unsorted inputs', + Buffer.from( + `${JSON.stringify( + changed(receipt(), (copy) => { + copy.inputs = { + 'package.json': 'b'.repeat(64), + 'apps/cli/package.json': 'a'.repeat(64), + }; + }), + null, + 2, + )}\n`, + ), + ], + ]) { + assert.throws(() => parseReleaseBinding(bad, synthetic), refuses('BINDING_BYTES'), name); + } + // Only a Buffer read from the file is parsed: a string or another view refuses. + assert.throws(() => parseReleaseBinding(text, synthetic), refuses('BINDING_BYTES')); + assert.throws( + () => parseReleaseBinding(new Uint8Array(bytes), synthetic), + refuses('BINDING_BYTES'), + ); + // A canonical receipt invalid for the expectation still refuses with its own code. + assert.throws( + () => parseReleaseBinding(bytes, { ...synthetic, packageVersion: '0.1.2' }), + refuses('BINDING_CANDIDATE'), + ); +}); + +test("the package version and fingerprint rules are binding.mjs's and parsePins'", () => { + /** @param {() => unknown} call */ + const admits = (call) => { + try { + call(); + return true; + } catch (error) { + if (error instanceof CustodyError) return false; + throw error; + } + }; + for (const version of [ + '0.1.1', + '1.2.3-rc.1', + '1.2.3+build.5', + '01.2.3', + '1.2', + 'v1.2.3', + '1.2.3-', + '1.2.3+', + `1.2.3-${'a'.repeat(94)}`, + `1.2.3-${'a'.repeat(95)}`, + ]) { + const value = changed(receipt(), (copy) => { + copy.packageVersion = version; + copy.tarball.name = tarballName(version); + copy.ciphertext.name = ciphertextFileName(version); + }); + assert.equal( + admits(() => validateReleaseBinding(value, { ...synthetic, packageVersion: version })), + isPackageVersion(version), + version, + ); + } + for (const fingerprint of [ + 'A'.repeat(40), + '0'.repeat(40), + 'a'.repeat(40), + 'A'.repeat(39), + 'A'.repeat(41), + 'G'.repeat(40), + `${'A'.repeat(39)}!`, + ]) { + const pins = { primary: fingerprint, subkey: PINS.subkey }; + assert.equal( + admits(() => validateReleaseBinding(receipt({ recipient: pins }), { ...synthetic, pins })), + admits(() => parsePins(pins)), + fingerprint, + ); + } +}); + +/** A retained candidate directory: its receipt, tarball and ciphertext. */ +function candidate({ + value = receipt(), + tarball = TARBALL, + ciphertext = CIPHERTEXT, + expected = synthetic, +} = {}) { + const directory = fresh('candidate'); + mkdirSync(directory); + writeFileSync( + join(directory, RELEASE_BINDING_FILE), + serializeReleaseBinding(validateReleaseBinding(value, expected)), + ); + writeFileSync(join(directory, tarballName(VERSION)), tarball); + writeFileSync(join(directory, ciphertextFileName(VERSION)), ciphertext); + return directory; +} +/** Every entry's kind and bytes, to show a refusal changed nothing; only regular files are read. */ +const snapshot = (directory) => + readdirSync(directory) + .sort() + .map((name) => { + const path = join(directory, name); + const info = lstatSync(path); + if (info.isFile()) return [name, sha256(readFileSync(path))]; + return [name, info.isSymbolicLink() ? `link ${readlinkSync(path)}` : String(info.mode)]; + }); + +test('a retained candidate is reused from its verified bytes, with no rebuild and no GnuPG call', () => { + for (const [value, expected] of [ + [receipt(), synthetic], + [productionReceipt(), productionExpected()], + ]) { + const directory = candidate({ value, expected }); + const before = snapshot(directory); + const verified = verifyRetainedCandidate({ directory, expected }); + assert.ok(Object.isFrozen(verified)); + assert.deepEqual(verified.receipt, value); + assert.deepEqual(verified.receiptBytes, readFileSync(join(directory, RELEASE_BINDING_FILE))); + assert.deepEqual(verified.tarball, TARBALL); + assert.deepEqual(verified.ciphertext, CIPHERTEXT); + // The bytes returned are the bytes verified: replacing a file afterwards changes nothing returned. + writeFileSync(join(directory, tarballName(VERSION)), Buffer.alloc(TARBALL.length, 0x78)); + assert.deepEqual(verified.tarball, TARBALL); + writeFileSync(join(directory, tarballName(VERSION)), TARBALL); + assert.deepEqual(snapshot(directory), before); + } + // A candidate sealed under a retired recipient is still reused after a rotation. + const retired = productionReceipt(A); + verifyRetainedCandidate({ + directory: candidate({ value: retired, expected: productionExpected() }), + expected: productionExpected(), + }); +}); + +test('a retained candidate whose assets or entries differ from its receipt refuses and is left as it was', () => { + const flip = (bytes) => { + const copy = Buffer.from(bytes); + copy[0] ^= 1; + return copy; + }; + const cases = [ + [ + 'CANDIDATE_DIGEST', + (directory) => writeFileSync(join(directory, tarballName(VERSION)), flip(TARBALL)), + ], + [ + 'CANDIDATE_DIGEST', + (directory) => writeFileSync(join(directory, ciphertextFileName(VERSION)), flip(CIPHERTEXT)), + ], + [ + 'CANDIDATE_SIZE', + (directory) => + writeFileSync( + join(directory, ciphertextFileName(VERSION)), + Buffer.concat([CIPHERTEXT, Buffer.from('x')]), + ), + ], + [ + 'CANDIDATE_SIZE', + (directory) => writeFileSync(join(directory, tarballName(VERSION)), TARBALL.subarray(1)), + ], + ['CANDIDATE_ENTRIES', (directory) => writeFileSync(join(directory, 'notes.txt'), 'extra')], + ['CANDIDATE_ENTRIES', (directory) => rmSync(join(directory, ciphertextFileName(VERSION)))], + ['CANDIDATE_ENTRIES', (directory) => rmSync(join(directory, RELEASE_BINDING_FILE))], + [ + 'CANDIDATE_ASSET', + (directory) => { + rmSync(join(directory, tarballName(VERSION))); + mkdirSync(join(directory, tarballName(VERSION))); + }, + ], + [ + 'BINDING_BYTES', + (directory) => + writeFileSync( + join(directory, RELEASE_BINDING_FILE), + Buffer.concat([readFileSync(join(directory, RELEASE_BINDING_FILE)), Buffer.from(' ')]), + ), + ], + ]; + if (posix) { + cases.push( + [ + 'CANDIDATE_ASSET', + (directory) => { + const target = fresh('linked-ciphertext'); + writeFileSync(target, CIPHERTEXT); + rmSync(join(directory, ciphertextFileName(VERSION))); + symlinkSync(target, join(directory, ciphertextFileName(VERSION))); + }, + ], + [ + 'CANDIDATE_ASSET', + (directory) => { + const target = fresh('linked-receipt'); + writeFileSync(target, readFileSync(join(directory, RELEASE_BINDING_FILE))); + rmSync(join(directory, RELEASE_BINDING_FILE)); + symlinkSync(target, join(directory, RELEASE_BINDING_FILE)); + }, + ], + [ + 'CANDIDATE_ASSET', + (directory) => { + rmSync(join(directory, tarballName(VERSION))); + assert.equal(spawnSync('mkfifo', [join(directory, tarballName(VERSION))]).status, 0); + }, + ], + ); + } + for (const [code, change] of cases) { + const directory = candidate(); + change(directory); + const before = snapshot(directory); + assert.throws( + () => verifyRetainedCandidate({ directory, expected: synthetic }), + refuses(code), + code, + ); + assert.deepEqual(snapshot(directory), before); + } + // A receipt over its bound is not read. + const large = candidate(); + writeFileSync(join(large, RELEASE_BINDING_FILE), Buffer.alloc(64 * 1024 + 1, 0x20)); + assert.throws( + () => verifyRetainedCandidate({ directory: large, expected: synthetic }), + refuses('CANDIDATE_ASSET'), + ); + // A stale receipt, for another candidate of the same version, refuses. + const stale = candidate({ value: productionReceipt(), expected: productionExpected() }); + assert.throws( + () => + verifyRetainedCandidate({ + directory: stale, + expected: { ...productionExpected(), source: 'f'.repeat(40) }, + }), + refuses('BINDING_CANDIDATE'), + ); + // So does a receipt verified in the wrong scope. + assert.throws( + () => verifyRetainedCandidate({ directory: stale, expected: synthetic }), + refuses('BINDING_SCOPE'), + ); +}); + +test('reuse needs a valid expectation, and a host failure refuses with a fixed code and no path', () => { + const directory = candidate(); + for (const [code, expected] of [ + ['BINDING_SCOPE', undefined], + ['BINDING_CANDIDATE', { ...synthetic, packageVersion: '../0.1.1' }], + ['BINDING_CANDIDATE', { ...synthetic, packageVersion: 1 }], + ]) { + assert.throws(() => verifyRetainedCandidate({ directory, expected }), refuses(code), code); + } + assert.throws( + () => + verifyRetainedCandidate({ directory: join(fresh('none'), 'candidate'), expected: synthetic }), + refuses('UNEXPECTED'), + ); + // The reuse API takes no GnuPG binary or run port. + assert.equal(verifyRetainedCandidate.length, 1); +}); From 4d1a189c977a03c5b434b99cd287a070c53d1cdc Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 21:04:17 +0300 Subject: [PATCH 80/97] feat(repo): gate encryption by scope and seal the binding receipt after the ciphertext encryptCandidate admits production scope only with the loaded recipient and a clean, non-synthetic candidate, and synthetic scope only with a synthetic candidate, before any GnuPG call; encryptOnce runs unchanged. sealReleaseBinding accepts only a record the gate returned and not yet sealed, refuses an existing receipt before reading anything, re-reads the ciphertext and the tarball against the record and the manifest, and publishes the receipt once through a hard link. A production round trip with an ephemeral key decrypts to the container the receipt binds. Refs: #46, ADR-0111 Co-Authored-By: Claude Opus 5.5 --- package.json | 2 +- tools/cli-package/release-custody.mjs | 261 +++++++++++ tools/cli-package/release-custody.test.mjs | 489 +++++++++++++++++++++ tools/cli-package/release-fixture.mjs | 82 ++++ 4 files changed, 833 insertions(+), 1 deletion(-) create mode 100644 tools/cli-package/release-custody.mjs create mode 100644 tools/cli-package/release-custody.test.mjs create mode 100644 tools/cli-package/release-fixture.mjs diff --git a/package.json b/package.json index 960bf20a..d793fbf4 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "smoke:budget-replay": "node tools/budget-replay-compat/check.mjs", "check:wire-mutations": "node tools/wire-conformance/check.mjs", "check:usage-mutations": "node tools/usage-coverage-controls/check.mjs", - "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/ustar.test.mjs tools/cli-package/container.test.mjs tools/cli-package/gnupg.test.mjs tools/cli-package/custody.test.mjs tools/cli-package/release-recipient.test.mjs tools/cli-package/release-binding.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/check.test.mjs tools/cli-package/paths.test.mjs tools/cli-package/source-map.test.mjs tools/cli-package/manifest.test.mjs tools/cli-package/symbolicate.test.mjs tools/cli-package/map-controls.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", + "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/ustar.test.mjs tools/cli-package/container.test.mjs tools/cli-package/gnupg.test.mjs tools/cli-package/custody.test.mjs tools/cli-package/release-recipient.test.mjs tools/cli-package/release-binding.test.mjs tools/cli-package/release-custody.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/check.test.mjs tools/cli-package/paths.test.mjs tools/cli-package/source-map.test.mjs tools/cli-package/manifest.test.mjs tools/cli-package/symbolicate.test.mjs tools/cli-package/map-controls.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", "check:fs-mutations": "node tools/fs-policy-controls/check.mjs", "check:secrets-mutations": "node tools/secrets-controls/check.mjs", "smoke:cli-bootstrap": "node tools/cli-bootstrap/check.mjs", diff --git a/tools/cli-package/release-custody.mjs b/tools/cli-package/release-custody.mjs new file mode 100644 index 00000000..d06ddee8 --- /dev/null +++ b/tools/cli-package/release-custody.mjs @@ -0,0 +1,261 @@ +/** + * Release custody (Step 9b.2; packaging.md, "Recipient custody implementation boundary — Step 9b.2"): the + * scope gate in front of the unchanged `encryptOnce`, and the release binding receipt's sealer. Production + * scope encrypts only a clean, non-synthetic candidate to the loaded production recipient; synthetic scope + * encrypts only a synthetic candidate, so neither a fixture key nor the per-run ephemeral key can ever + * encrypt a product staging. The sealer accepts only a record this gate returned, and writes the receipt + * once, after re-reading the ciphertext it names. This is the only release module that reaches GnuPG. + */ +import { randomUUID } from 'node:crypto'; +import { + closeSync, + fsyncSync, + linkSync, + lstatSync, + openSync, + readdirSync, + realpathSync, + rmSync, + writeSync, +} from 'node:fs'; +import { join } from 'node:path'; +import { isDeepStrictEqual } from 'node:util'; +import { ARCHIVE_LIMITS } from './archive.mjs'; +import { readPrivateContainer } from './container.mjs'; +import { CustodyError, refuseCustody } from './custody-error.mjs'; +import { encryptOnce } from './custody.mjs'; +import { readOrdinaryFile, sha256 } from './files.mjs'; +import { + RELEASE_BINDING_FILE, + RELEASE_BINDING_KIND, + serializeReleaseBinding, + tarballName, + validateReleaseBinding, +} from './release-binding.mjs'; +import { isProductionRecipient, isSyntheticSource } from './release-recipient.mjs'; + +/** + * @typedef {import('./custody.mjs').CustodyRecord} CustodyRecord + * @typedef {import('./release-recipient.mjs').ProductionRecipient} ProductionRecipient + * @typedef {{kind: 'production', recipient: ProductionRecipient} + * | {kind: 'synthetic', recipient: {file: string, pins: unknown}}} Scope + */ + +/** + * The records the gate returned, with what the sealer needs to bind them. + * @type {WeakMap} + */ +const records = new WeakMap(); + +/** @param {unknown} value @returns {value is Record} */ +const isRecord = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); + +/** Whether `path` names an entry, a dangling link included. @param {string} path */ +function present(path) { + try { + lstatSync(path); + return true; + } catch { + return false; + } +} + +/** + * Encrypt a candidate's container once for its scope. Every refusal here comes before any GnuPG call; + * `encryptOnce`'s own refusals follow unchanged. + * @param {{scope: Scope, container: Buffer, destination: string, privateRoot: string, now?: number, + * run?: typeof import('./process.mjs').runCommand, gpg?: import('./gnupg.mjs').GnupgInstallation}} + * options + * @returns {Promise} + */ +export async function encryptCandidate({ + scope, + container, + destination, + privateRoot, + now, + run, + gpg, +}) { + let admitted; + try { + admitted = admit(scope, container, destination); + } catch (error) { + // A host error's message names a path: only fixed custody codes leave. + if (error instanceof CustodyError) throw error; + refuseCustody('UNEXPECTED'); + } + const record = await encryptOnce({ + container, + recipient: admitted.recipient, + destination, + privateRoot, + now, + run, + gpg, + }); + try { + records.set(record, { + scope, + manifest: admitted.manifest, + destination: realpathSync(destination), + }); + } catch { + refuseCustody('UNEXPECTED'); + } + return record; +} + +/** @param {unknown} scope @param {Buffer} container @param {string} destination */ +function admit(scope, container, destination) { + if ( + !isRecord(scope) || + (scope.kind !== 'production' && scope.kind !== 'synthetic') || + !isRecord(scope.recipient) + ) { + refuseCustody('SCOPE'); + } + if (scope.kind === 'production' && !isProductionRecipient(scope.recipient)) { + refuseCustody('RECIPIENT_SOURCE'); + } + const { manifest } = readPrivateContainer(container); + let recipient; + if (scope.kind === 'production') { + if (manifest.source.dirty) refuseCustody('DIRTY_SOURCE'); + if (isSyntheticSource(manifest.source)) refuseCustody('SYNTHETIC_SOURCE'); + const production = /** @type {ProductionRecipient} */ (scope.recipient); + recipient = { file: production.file, pins: production.current }; + } else { + if (!isSyntheticSource(manifest.source)) refuseCustody('FIXTURE_SCOPE'); + recipient = /** @type {{file: string, pins: unknown}} */ (scope.recipient); + } + if (present(join(destination, RELEASE_BINDING_FILE))) refuseCustody('ALREADY_SEALED'); + return { manifest, recipient }; +} + +/** Write a new public file durably; an existing file refuses. @param {string} path @param {Buffer} bytes */ +function writePublicFile(path, bytes) { + const descriptor = openSync(path, 'wx', 0o644); + try { + let offset = 0; + while (offset < bytes.length) offset += writeSync(descriptor, bytes, offset); + fsyncSync(descriptor); + } finally { + closeSync(descriptor); + } +} + +/** + * Publish once and atomically: the bytes go to a fresh sibling, which is then hard-linked to the final + * name. A link never replaces an existing receipt, and a failed write never leaves a partial file under + * the final name. Once the link exists the receipt is published: a sibling that cannot be removed then + * is left behind rather than reported as a refusal. + * @param {string} directory @param {Buffer} bytes @param {(path: string, bytes: Buffer) => void} write + */ +function publish(directory, bytes, write) { + const temporary = join(directory, `.${RELEASE_BINDING_FILE}.${randomUUID()}.partial`); + try { + write(temporary, bytes); + try { + linkSync(temporary, join(directory, RELEASE_BINDING_FILE)); + } catch (error) { + if (error instanceof Error && 'code' in error && error.code === 'EEXIST') { + refuseCustody('ALREADY_SEALED'); + } + throw error; + } + } finally { + try { + rmSync(temporary, { force: true }); + } catch { + // Best effort: the final name alone is the published claim. + } + } +} + +/** + * Seal the release binding receipt for a record `encryptCandidate` returned, beside its ciphertext and + * the candidate's tarball. `origin` is the originating trusted run (Step 9d supplies and binds it); `write` + * is a test seam for fault injection. + * @param {CustodyRecord} record + * @param {{origin?: unknown, write?: (path: string, bytes: Buffer) => void}} [options] + * @returns {{receipt: ReturnType, bytes: Buffer}} + */ +export function sealReleaseBinding(record, { origin, write = writePublicFile } = {}) { + try { + return seal(record, origin, write); + } catch (error) { + if (error instanceof CustodyError) throw error; + return refuseCustody('UNEXPECTED'); + } +} + +/** + * @param {CustodyRecord} record @param {unknown} origin + * @param {(path: string, bytes: Buffer) => void} write + */ +function seal(record, origin, write) { + const entry = records.get(record); + if (entry === undefined) refuseCustody('RECORD_SOURCE'); + const { scope, manifest, destination } = entry; + // An existing receipt refuses before anything is read; one is never written over. + if (present(join(destination, RELEASE_BINDING_FILE))) refuseCustody('ALREADY_SEALED'); + const tarball = tarballName(manifest.packageVersion); + if (!isDeepStrictEqual(readdirSync(destination).sort(), [record.name, tarball].sort())) { + refuseCustody('CANDIDATE_ENTRIES'); + } + // The ciphertext must still be the bytes encrypted: the receipt is sealed only after it exists. + let ciphertext; + try { + ciphertext = readOrdinaryFile(join(destination, record.name), record.bytes); + } catch { + refuseCustody('CIPHERTEXT_DRIFT'); + } + if (ciphertext.length !== record.bytes || sha256(ciphertext) !== record.sha256) { + refuseCustody('CIPHERTEXT_DRIFT'); + } + let tarballBytes; + try { + tarballBytes = readOrdinaryFile(join(destination, tarball), ARCHIVE_LIMITS.compressedBytes); + } catch { + refuseCustody('TARBALL_DRIFT'); + } + const tarballSha256 = sha256(tarballBytes); + if (tarballSha256 !== manifest.tarballSha256) refuseCustody('TARBALL_DRIFT'); + + const expected = + scope.kind === 'production' + ? { + scope: /** @type {const} */ ('production'), + recipient: scope.recipient, + packageVersion: manifest.packageVersion, + source: manifest.source.commit, + } + : { + scope: /** @type {const} */ ('synthetic'), + pins: record.recipient, + packageVersion: manifest.packageVersion, + }; + const receipt = validateReleaseBinding( + { + version: 1, + kind: RELEASE_BINDING_KIND, + packageVersion: manifest.packageVersion, + source: { ...manifest.source }, + tarball: { name: tarball, bytes: tarballBytes.length, sha256: tarballSha256 }, + ciphertext: { name: record.name, bytes: record.bytes, sha256: record.sha256 }, + recipient: { ...record.recipient }, + toolchain: { ...manifest.toolchain, gnupg: record.gnupg }, + target: manifest.target, + inputs: { ...manifest.inputs }, + mapCheck: { status: 'complete' }, + origin, + }, + expected, + ); + const bytes = serializeReleaseBinding(receipt); + publish(destination, bytes, write); + records.delete(record); + return Object.freeze({ receipt, bytes }); +} diff --git a/tools/cli-package/release-custody.test.mjs b/tools/cli-package/release-custody.test.mjs new file mode 100644 index 00000000..6fcde1be --- /dev/null +++ b/tools/cli-package/release-custody.test.mjs @@ -0,0 +1,489 @@ +import assert from 'node:assert/strict'; +import { Buffer } from 'node:buffer'; +import { + appendFileSync, + chmodSync, + mkdirSync, + readFileSync, + readdirSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { createRequire, syncBuiltinESMExports } from 'node:module'; +import { join } from 'node:path'; +import { before, test } from 'node:test'; +import { fresh, posixModes } from './candidate-fixture.mjs'; +import { readPrivateContainer } from './container.mjs'; +import { CustodyError } from './custody-error.mjs'; +import { encryptOnce } from './custody.mjs'; +import { sha256 } from './files.mjs'; +import { PINS, ephemeralKey, fixture } from './gpg-fixture.mjs'; +import { runCommand } from './process.mjs'; +import { + RELEASE_BINDING_FILE, + ciphertextFileName, + parseReleaseBinding, + serializeReleaseBinding, + tarballName, + verifyRetainedCandidate, +} from './release-binding.mjs'; +import { encryptCandidate, sealReleaseBinding } from './release-custody.mjs'; +import { + CLEAN, + TARBALL, + VERSION, + containerWith, + containers, + origin, + places, + recipientRoot, +} from './release-fixture.mjs'; +import { SYNTHETIC_SOURCE, loadProductionRecipient } from './release-recipient.mjs'; + +const refuses = (code) => (error) => + error instanceof CustodyError && + error.code === code && + /^PACKED_CLI_CUSTODY_[A-Z_]+$/u.test(error.message); + +/** @type {{staging: string, synthetic: Buffer, clean: Buffer}} */ +let built; +before(async () => { + built = await containers(); +}); + +/** A run port that records every GnuPG call; everything else is the real GnuPG. */ +function recording() { + const calls = []; + /** @type {typeof runCommand} */ + const run = async (command, args, options) => { + calls.push( + args.find( + (arg) => + arg.startsWith('--') && + ![ + '--homedir', + '--batch', + '--no-tty', + '--no-options', + '--no-autostart', + '--no-keyring', + ].includes(arg), + ), + ); + return runCommand(command, args, options); + }; + return { run, calls }; +} +const synthetic = Object.freeze({ + kind: 'synthetic', + recipient: { file: fixture('good'), pins: PINS.good }, +}); + +/** No receipt of either kind exists: custody never downgrades to a package-test artifact. */ +function noReceipt(destination) { + const names = readdirSync(destination); + assert.equal(names.includes(RELEASE_BINDING_FILE), false, 'a release receipt was written'); + assert.equal( + names.includes('package-test-receipt.json'), + false, + 'a package-test receipt was written', + ); +} + +test('the gate refuses a wrong scope, recipient or candidate before any GnuPG call', async () => { + const unbranded = { + kind: 'production', + recipient: { file: fixture('good'), current: PINS.good, history: [] }, + }; + const ephemeral = ephemeralKey(); + const production = { + kind: 'production', + recipient: loadProductionRecipient({ + root: recipientRoot(ephemeral.file, [{ ...ephemeral.pins, status: 'current' }]), + }), + }; + const dirtyClean = containerWith(built.staging, { ...CLEAN, dirty: true }); + const cleanSentinel = containerWith(built.staging, { ...SYNTHETIC_SOURCE, dirty: false }); + for (const [code, scope, container] of [ + ['SCOPE', undefined, built.synthetic], + ['SCOPE', { kind: 'release', recipient: synthetic.recipient }, built.synthetic], + ['SCOPE', { kind: 'synthetic' }, built.synthetic], + ['RECIPIENT_SOURCE', unbranded, built.clean], + ['RECIPIENT_SOURCE', { ...production, recipient: { ...production.recipient } }, built.clean], + ['DIRTY_SOURCE', production, dirtyClean], + ['DIRTY_SOURCE', production, built.synthetic], + ['SYNTHETIC_SOURCE', production, cleanSentinel], + ['FIXTURE_SCOPE', synthetic, built.clean], + ['FIXTURE_SCOPE', synthetic, dirtyClean], + ]) { + const { root, destination } = places(); + const { run, calls } = recording(); + await assert.rejects( + encryptCandidate({ scope, container, destination, privateRoot: root, run }), + refuses(code), + code, + ); + assert.deepEqual(calls, [], code); + assert.deepEqual(readdirSync(destination), [tarballName(VERSION)], code); + } + // A destination that already holds a receipt is sealed: nothing is encrypted into it. + for (const kind of ['file', 'dangling link']) { + const { root, destination } = places(); + if (kind === 'file') writeFileSync(join(destination, RELEASE_BINDING_FILE), '{}'); + else if (posixModes) + symlinkSync(join(destination, 'missing'), join(destination, RELEASE_BINDING_FILE)); + else continue; + const { run, calls } = recording(); + await assert.rejects( + encryptCandidate({ + scope: synthetic, + container: built.synthetic, + destination, + privateRoot: root, + run, + }), + refuses('ALREADY_SEALED'), + ); + assert.deepEqual(calls, []); + } + // A host failure in the gate refuses with a fixed code, never the host's error. + { + const { root } = places(); + const { run, calls } = recording(); + await assert.rejects( + encryptCandidate({ + scope: synthetic, + container: built.synthetic, + destination: undefined, + privateRoot: root, + run, + }), + refuses('UNEXPECTED'), + ); + assert.deepEqual(calls, []); + } + // A container that is not one refuses as encryptOnce would, before any call. + const { root, destination } = places(); + const { run, calls } = recording(); + await assert.rejects( + encryptCandidate({ + scope: synthetic, + container: Buffer.from('x'), + destination, + privateRoot: root, + run, + }), + (error) => error instanceof CustodyError && error.code.startsWith('CONTAINER_'), + ); + assert.deepEqual(calls, []); +}); + +/** Encrypt the synthetic container in fresh places and return everything the sealer tests need. */ +async function encrypted(scope = synthetic, container = built.synthetic) { + const { root, destination } = places(); + const record = await encryptCandidate({ scope, container, destination, privateRoot: root }); + return { root, destination, record }; +} + +test('a synthetic candidate is encrypted, then sealed once beside its ciphertext and tarball', async () => { + const { root, destination, record } = await encrypted(); + assert.deepEqual(record.recipient, PINS.good); + assert.deepEqual(readdirSync(root), []); + const sealed = sealReleaseBinding(record, { origin: origin(SYNTHETIC_SOURCE.commit) }); + assert.deepEqual( + readdirSync(destination).sort(), + [ciphertextFileName(VERSION), RELEASE_BINDING_FILE, tarballName(VERSION)].sort(), + ); + const bytes = readFileSync(join(destination, RELEASE_BINDING_FILE)); + assert.deepEqual(bytes, sealed.bytes); + assert.deepEqual(serializeReleaseBinding(sealed.receipt), bytes); + const manifest = readPrivateContainer(built.synthetic).manifest; + const expected = { scope: 'synthetic', pins: PINS.good, packageVersion: VERSION }; + assert.deepEqual(parseReleaseBinding(bytes, expected), sealed.receipt); + assert.deepEqual(sealed.receipt.source, manifest.source); + assert.deepEqual(sealed.receipt.tarball, { + name: tarballName(VERSION), + bytes: TARBALL.length, + sha256: sha256(TARBALL), + }); + assert.deepEqual(sealed.receipt.ciphertext, { + name: record.name, + bytes: record.bytes, + sha256: record.sha256, + }); + assert.deepEqual(sealed.receipt.recipient, PINS.good); + assert.equal(sealed.receipt.toolchain.gnupg, record.gnupg); + assert.equal(sealed.receipt.toolchain.node, manifest.toolchain.node); + assert.deepEqual(sealed.receipt.inputs, manifest.inputs); + assert.equal(sealed.receipt.target, manifest.target); + // The sealed candidate is reusable as it stands, and a record seals only once. + verifyRetainedCandidate({ directory: destination, expected }); + assert.throws( + () => sealReleaseBinding(record, { origin: origin(SYNTHETIC_SOURCE.commit) }), + refuses('RECORD_SOURCE'), + ); +}); + +test('only a record the gate returned seals, and an existing receipt refuses before anything is read', async () => { + const { destination, record } = await encrypted(); + const direct = places(); + const unscoped = await encryptOnce({ + container: built.synthetic, + recipient: synthetic.recipient, + destination: direct.destination, + privateRoot: direct.root, + }); + for (const value of [ + { ...record }, + globalThis.structuredClone(record), + unscoped, + undefined, + null, + 'record', + ]) { + assert.throws( + () => sealReleaseBinding(value, { origin: origin(SYNTHETIC_SOURCE.commit) }), + refuses('RECORD_SOURCE'), + ); + } + // A planted receipt wins over the unreadable ciphertext: nothing was read. + writeFileSync(join(destination, RELEASE_BINDING_FILE), 'planted'); + rmSync(join(destination, record.name)); + assert.throws( + () => sealReleaseBinding(record, { origin: origin(SYNTHETIC_SOURCE.commit) }), + refuses('ALREADY_SEALED'), + ); + assert.equal(readFileSync(join(destination, RELEASE_BINDING_FILE), 'utf8'), 'planted'); + // Calling without options refuses with a fixed code, never a host error. + const again = await encrypted(); + assert.throws(() => sealReleaseBinding(again.record), refuses('BINDING_ORIGIN')); + noReceipt(again.destination); +}); + +test('the sealer refuses a candidate directory that is not exactly its tarball and the ciphertext encrypted', async () => { + const flip = (path) => { + const bytes = readFileSync(path); + bytes[bytes.length - 1] ^= 1; + writeFileSync(path, bytes); + }; + const cases = [ + ['CANDIDATE_ENTRIES', (destination) => writeFileSync(join(destination, 'notes.txt'), 'x')], + ['CANDIDATE_ENTRIES', (destination) => rmSync(join(destination, tarballName(VERSION)))], + ['CANDIDATE_ENTRIES', (destination, record) => rmSync(join(destination, record.name))], + [ + 'CIPHERTEXT_DRIFT', + (destination, record) => appendFileSync(join(destination, record.name), 'x'), + ], + ['CIPHERTEXT_DRIFT', (destination, record) => flip(join(destination, record.name))], + [ + 'CIPHERTEXT_DRIFT', + (destination, record) => { + const bytes = readFileSync(join(destination, record.name)); + writeFileSync(join(destination, record.name), bytes.subarray(1)); + }, + ], + ['TARBALL_DRIFT', (destination) => flip(join(destination, tarballName(VERSION)))], + [ + 'TARBALL_DRIFT', + (destination) => + writeFileSync(join(destination, tarballName(VERSION)), Buffer.alloc(16 * 1024 * 1024 + 1)), + ], + ]; + if (posixModes) { + cases.push( + [ + 'CIPHERTEXT_DRIFT', + (destination, record) => { + const target = fresh('linked-ciphertext'); + writeFileSync(target, readFileSync(join(destination, record.name))); + rmSync(join(destination, record.name)); + symlinkSync(target, join(destination, record.name)); + }, + ], + [ + 'TARBALL_DRIFT', + (destination) => { + const target = fresh('linked-tarball'); + writeFileSync(target, TARBALL); + rmSync(join(destination, tarballName(VERSION))); + symlinkSync(target, join(destination, tarballName(VERSION))); + }, + ], + ); + } + for (const [code, change] of cases) { + const { destination, record } = await encrypted(); + change(destination, record); + assert.throws( + () => sealReleaseBinding(record, { origin: origin(SYNTHETIC_SOURCE.commit) }), + refuses(code), + code, + ); + noReceipt(destination); + } + // A malformed origin refuses in the self-check, before anything is written. + for (const value of [ + { kind: 'local' }, + origin('f'.repeat(40)), + { ...origin(SYNTHETIC_SOURCE.commit), event: 'pull_request' }, + ]) { + const { destination, record } = await encrypted(); + assert.throws(() => sealReleaseBinding(record, { origin: value }), refuses('BINDING_ORIGIN')); + noReceipt(destination); + } +}); + +test('the receipt is published once and atomically, whatever happens to the write', async () => { + // A receipt that appears while sealing is never overwritten. + const raced = await encrypted(); + assert.throws( + () => + sealReleaseBinding(raced.record, { + origin: origin(SYNTHETIC_SOURCE.commit), + write: (path, bytes) => { + writeFileSync(join(raced.destination, RELEASE_BINDING_FILE), 'raced'); + writeFileSync(path, bytes); + }, + }), + refuses('ALREADY_SEALED'), + ); + assert.equal(readFileSync(join(raced.destination, RELEASE_BINDING_FILE), 'utf8'), 'raced'); + assert.deepEqual( + readdirSync(raced.destination).sort(), + [ciphertextFileName(VERSION), RELEASE_BINDING_FILE, tarballName(VERSION)].sort(), + ); + // A write that fails midway leaves nothing under the final name and no partial sibling. + const failed = await encrypted(); + assert.throws( + () => + sealReleaseBinding(failed.record, { + origin: origin(SYNTHETIC_SOURCE.commit), + write: (path, bytes) => { + writeFileSync(path, bytes.subarray(0, 10)); + throw new Error(`disk full at ${path}`); + }, + }), + (error) => refuses('UNEXPECTED')(error) && !error.message.includes(failed.destination), + ); + assert.deepEqual( + readdirSync(failed.destination).sort(), + [ciphertextFileName(VERSION), tarballName(VERSION)].sort(), + ); + // The record survives a failed seal, so the candidate can still be sealed. + sealReleaseBinding(failed.record, { origin: origin(SYNTHETIC_SOURCE.commit) }); + assert.ok(readdirSync(failed.destination).includes(RELEASE_BINDING_FILE)); +}); + +test( + 'a sibling that cannot be removed after the link leaves the receipt published, never a false refusal', + { skip: !posixModes || process.getuid?.() === 0 }, + async () => { + const { destination, record } = await encrypted(); + const fs = createRequire(import.meta.url)('node:fs'); + const link = fs.linkSync; + fs.linkSync = (existing, created) => { + link(existing, created); + chmodSync(destination, 0o500); + }; + syncBuiltinESMExports(); + try { + const sealed = sealReleaseBinding(record, { origin: origin(SYNTHETIC_SOURCE.commit) }); + assert.deepEqual(readFileSync(join(destination, RELEASE_BINDING_FILE)), sealed.bytes); + assert.equal(readdirSync(destination).filter((name) => name.endsWith('.partial')).length, 1); + } finally { + fs.linkSync = link; + syncBuiltinESMExports(); + chmodSync(destination, 0o700); + } + }, +); + +test('a host failure after encryption refuses with a fixed code and no path', async () => { + const { root, destination } = places(); + const fs = createRequire(import.meta.url)('node:fs'); + const realpath = fs.realpathSync; + let calls = 0; + // encryptOnce resolves the destination once; the gate's second resolution fails. + fs.realpathSync = Object.assign( + (path, ...rest) => { + if (path === destination && (calls += 1) === 2) throw new Error(`vanished: ${destination}`); + return realpath(path, ...rest); + }, + { native: realpath.native }, + ); + syncBuiltinESMExports(); + try { + await assert.rejects( + encryptCandidate({ + scope: synthetic, + container: built.synthetic, + destination, + privateRoot: root, + }), + (error) => refuses('UNEXPECTED')(error) && !error.message.includes(destination), + ); + } finally { + fs.realpathSync = realpath; + syncBuiltinESMExports(); + } + assert.equal(calls, 2); +}); + +test('a production candidate round-trips: the receipt binds what the ciphertext decrypts to', async () => { + const ephemeral = ephemeralKey(); + // A rotated history: an earlier recipient retired, the ephemeral key current. + const earlier = { primary: '1'.repeat(40), subkey: '2'.repeat(40) }; + const recipient = loadProductionRecipient({ + root: recipientRoot(ephemeral.file, [ + { ...earlier, status: 'retired' }, + { ...ephemeral.pins, status: 'current' }, + ]), + }); + const { destination, record } = await encrypted({ kind: 'production', recipient }, built.clean); + assert.deepEqual(record.recipient, ephemeral.pins); + const sealed = sealReleaseBinding(record, { origin: origin(CLEAN.commit) }); + const out = fresh('release-round-trip'); + mkdirSync(out); + ephemeral.decrypt(join(destination, record.name), join(out, 'container.tar')); + const decrypted = readFileSync(join(out, 'container.tar')); + assert.deepEqual(decrypted, built.clean); + const { manifest } = readPrivateContainer(decrypted); + const { receipt } = sealed; + assert.equal(receipt.packageVersion, manifest.packageVersion); + assert.deepEqual(receipt.source, manifest.source); + assert.equal(receipt.tarball.sha256, manifest.tarballSha256); + const { gnupg, ...tools } = receipt.toolchain; + assert.deepEqual(tools, manifest.toolchain); + assert.equal(gnupg, record.gnupg); + assert.equal(receipt.target, manifest.target); + assert.deepEqual(receipt.inputs, manifest.inputs); + const expected = { + scope: 'production', + recipient, + packageVersion: VERSION, + source: CLEAN.commit, + }; + const reused = verifyRetainedCandidate({ directory: destination, expected }); + assert.deepEqual(reused.receipt, receipt); + // The production receipt never verifies as synthetic, nor under a history without its recipient. + assert.throws( + () => + verifyRetainedCandidate({ + directory: destination, + expected: { scope: 'synthetic', pins: ephemeral.pins, packageVersion: VERSION }, + }), + refuses('BINDING_SCOPE'), + ); + const rotatedAway = loadProductionRecipient({ + root: recipientRoot(ephemeral.file, [{ ...earlier, status: 'current' }]), + }); + assert.throws( + () => + verifyRetainedCandidate({ + directory: destination, + expected: { ...expected, recipient: rotatedAway }, + }), + refuses('BINDING_RECIPIENT'), + ); +}); diff --git a/tools/cli-package/release-fixture.mjs b/tools/cli-package/release-fixture.mjs new file mode 100644 index 00000000..a769ded2 --- /dev/null +++ b/tools/cli-package/release-fixture.mjs @@ -0,0 +1,82 @@ +/** + * Test-only release candidates (Step 9b.2). The synthetic sealed staging of `candidate-fixture.mjs`, sealed + * with a real tarball digest, gives a synthetic-scope container; a copy whose manifest records a clean, + * non-synthetic source gives a production-scope one (the container checks structure, not provenance). + * Production-scope tests load a temporary recipient root, never the committed path, and never a fixture + * key. Nothing here is a product; only tests import it. + */ +import { Buffer } from 'node:buffer'; +import { chmodSync, copyFileSync, cpSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { candidate, digest, fresh, seal } from './candidate-fixture.mjs'; +import { buildPrivateContainer } from './container.mjs'; + +/** A synthetic tarball: its bytes are never inspected, only named and hashed. */ +export const TARBALL = Buffer.from('a synthetic tarball, never a package\n'); +export const VERSION = '0.1.1'; +/** The clean, non-synthetic source the production-scope container records. */ +export const CLEAN = Object.freeze({ commit: 'd'.repeat(40), tree: 'e'.repeat(40) }); + +/** + * A container of a copy of `staging` whose manifest records `source`. + * @param {string} staging @param {{commit: string, tree: string, dirty: boolean}} source + */ +export function containerWith(staging, source) { + const copy = fresh('release-staging'); + cpSync(staging, copy, { recursive: true }); + const manifest = JSON.parse(readFileSync(join(copy, 'manifest.json'), 'utf8')); + manifest.source = source; + writeFileSync(join(copy, 'manifest.json'), JSON.stringify(manifest)); + return buildPrivateContainer(copy).bytes; +} + +/** + * The sealed synthetic staging, its container, and a clean one, all naming `TARBALL`'s digest. + * @returns {Promise<{staging: string, synthetic: Buffer, clean: Buffer}>} + */ +export async function containers() { + const fixture = await candidate(); + fixture.archive.sha256 = digest(TARBALL); + await seal(fixture); + writeFileSync(join(fixture.staging, 'metafile-esm.json'), '{"inputs":{},"outputs":{}}'); + return { + staging: fixture.staging, + synthetic: buildPrivateContainer(fixture.staging).bytes, + clean: containerWith(fixture.staging, { ...CLEAN, dirty: false }), + }; +} + +/** A fresh owner-only private root and a destination holding the candidate's tarball. */ +export function places() { + const root = fresh('release-root'); + mkdirSync(root); + chmodSync(root, 0o700); + const destination = fresh('release-destination'); + mkdirSync(destination); + writeFileSync(join(destination, `relavium-${VERSION}.tgz`), TARBALL); + return { root, destination }; +} + +/** A synthetic originating GitHub push run for `source`. @param {string} source */ +export const origin = (source) => ({ + kind: 'github', + repository: 'example/relavium', + workflow: 'release', + runId: '1', + runAttempt: '1', + event: 'push', + source, +}); + +/** + * A temporary production recipient root holding `file` and the given history. + * @param {string} file + * @param {{primary: string, subkey: string, status: string}[]} recipients + */ +export function recipientRoot(file, recipients) { + const root = fresh('release-recipient'); + mkdirSync(root); + copyFileSync(file, join(root, 'recipient.asc')); + writeFileSync(join(root, 'pins.json'), JSON.stringify({ version: 1, recipients })); + return root; +} From 4d49176bb9e64554c3c4f038879e9055c8cd7cfd Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 21:04:17 +0300 Subject: [PATCH 81/97] test(repo): pin release scope separation, the reuse closure and the interim guard No product or tooling code reaches a test fixture, only the loader reads the fixture pins, only the release gate imports encryptOnce, the receipt module's import closure starts no process, the interim guard still refuses a complete synthetic candidate, and the restated constants equal their sources. Until Step 9d no product path or workflow names a release module, and until Wave 4b no recipient directory is committed. Refs: #46, ADR-0111 Co-Authored-By: Claude Opus 5.5 --- package.json | 2 +- tools/cli-package/release-scope.test.mjs | 167 +++++++++++++++++++++++ 2 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 tools/cli-package/release-scope.test.mjs diff --git a/package.json b/package.json index d793fbf4..6f22957e 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "smoke:budget-replay": "node tools/budget-replay-compat/check.mjs", "check:wire-mutations": "node tools/wire-conformance/check.mjs", "check:usage-mutations": "node tools/usage-coverage-controls/check.mjs", - "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/ustar.test.mjs tools/cli-package/container.test.mjs tools/cli-package/gnupg.test.mjs tools/cli-package/custody.test.mjs tools/cli-package/release-recipient.test.mjs tools/cli-package/release-binding.test.mjs tools/cli-package/release-custody.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/check.test.mjs tools/cli-package/paths.test.mjs tools/cli-package/source-map.test.mjs tools/cli-package/manifest.test.mjs tools/cli-package/symbolicate.test.mjs tools/cli-package/map-controls.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", + "test:acceptance-tools": "node --test tools/ci/windows-headless-smoke.test.mjs tools/ci/workspace-scheduling.test.mjs tools/ci/require-native-db-diagnostics.test.mjs tools/usage-coverage-controls/outcome.test.mjs tools/usage-coverage-controls/workload.test.mjs tools/ci/require-w8-acceptance.test.mjs tools/acceptance-evidence/export.test.mjs tools/wire-conformance/provenance.test.mjs tools/causal-controls/outcome.test.mjs tools/causal-controls/process.test.mjs tools/causal-controls/provenance.test.mjs tools/causal-controls/retention.test.mjs tools/causal-controls/run.test.mjs tools/causal-controls/timeout-policy.test.mjs tools/bundle-closure/closure.test.mjs tools/cli-bootstrap/witness.test.mjs tools/ci/require-packed-cli.test.mjs tools/cli-package/archive.test.mjs tools/cli-package/ustar.test.mjs tools/cli-package/container.test.mjs tools/cli-package/gnupg.test.mjs tools/cli-package/custody.test.mjs tools/cli-package/release-recipient.test.mjs tools/cli-package/release-binding.test.mjs tools/cli-package/release-custody.test.mjs tools/cli-package/release-scope.test.mjs tools/cli-package/seal.test.mjs tools/cli-package/binding.test.mjs tools/cli-package/process.test.mjs tools/cli-package/files.test.mjs tools/cli-package/smoke/process.test.mjs tools/cli-package/smoke/preload.test.mjs tools/cli-package/smoke/failure-diagnostic.test.mjs tools/cli-package/smoke/native-process.test.mjs tools/cli-package/smoke/runtime-stderr.test.mjs tools/cli-package/smoke/native-spawn-witness.test.mjs tools/cli-package/pipeline.test.mjs tools/cli-package/finalize.test.mjs tools/cli-package/candidate.test.mjs tools/cli-package/prepack.test.mjs tools/cli-package/release-guard.test.mjs tools/cli-package/produce.test.mjs tools/cli-package/check.test.mjs tools/cli-package/paths.test.mjs tools/cli-package/source-map.test.mjs tools/cli-package/manifest.test.mjs tools/cli-package/symbolicate.test.mjs tools/cli-package/map-controls.test.mjs tools/cli-package/smoke/installation.test.mjs tools/cli-package/consumer-lifetime.test.mjs", "check:fs-mutations": "node tools/fs-policy-controls/check.mjs", "check:secrets-mutations": "node tools/secrets-controls/check.mjs", "smoke:cli-bootstrap": "node tools/cli-bootstrap/check.mjs", diff --git a/tools/cli-package/release-scope.test.mjs b/tools/cli-package/release-scope.test.mjs new file mode 100644 index 00000000..22d90236 --- /dev/null +++ b/tools/cli-package/release-scope.test.mjs @@ -0,0 +1,167 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join, posix, relative } from 'node:path'; +import { test } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { COMMIT, TREE, fresh } from './candidate-fixture.mjs'; +import { CONTAINER_LIMITS } from './container.mjs'; +import { ciphertextName } from './custody.mjs'; +import { + CIPHERTEXT_BYTES, + RELEASE_BINDING_FILE, + ciphertextFileName, + tarballName, +} from './release-binding.mjs'; +import { ReleaseGuardError, requireReleaseCandidate } from './release-guard-core.mjs'; +import { PRODUCTION_RECIPIENT_ROOT, SYNTHETIC_SOURCE } from './release-recipient.mjs'; + +const repository = fileURLToPath(new URL('../../', import.meta.url)); +const here = fileURLToPath(new URL('.', import.meta.url)); +/** Tracked and new, unignored source files, as repository-relative POSIX paths. */ +const tracked = spawnSync( + 'git', + [ + 'ls-files', + '-z', + '--cached', + '--others', + '--exclude-standard', + '--', + '*.mjs', + '*.cjs', + '*.js', + '*.ts', + '*.tsx', + '*.mts', + '*.yml', + '*.yaml', + ], + { cwd: repository, encoding: 'utf8' }, +) + .stdout.split('\0') + .filter(Boolean); +/** Test, fixture and maintainer-only files: everything else is product or tooling code. */ +const isTestOnly = (path) => + /\.test\.[cm]?[jt]sx?$/u.test(path) || + /-fixture\.mjs$/u.test(path) || + /(?:^|\/)(?:test|tests|__tests__|lint-fixtures)\//u.test(path) || + path.startsWith('tools/cli-package/custody-fixtures/'); +const code = tracked.filter((path) => !isTestOnly(path)); +const read = (path) => readFileSync(join(repository, path), 'utf8'); +/** The code files whose text matches `pattern`. @param {RegExp} pattern */ +const naming = (pattern) => code.filter((path) => pattern.test(read(path))).sort(); + +test('the scan sees the custody modules and leaves the tests out', () => { + for (const path of [ + 'tools/cli-package/custody.mjs', + 'tools/cli-package/release-custody.mjs', + 'tools/cli-package/release-guard-core.mjs', + '.github/workflows/release.yml', + ]) { + assert.ok(code.includes(path), path); + } + for (const path of [ + 'tools/cli-package/release-custody.test.mjs', + 'tools/cli-package/release-fixture.mjs', + 'tools/cli-package/custody-fixtures/generate.mjs', + ]) { + assert.ok(tracked.includes(path) && !code.includes(path), path); + } +}); + +test('no product or tooling code reaches a test fixture, and only the loader reads the fixture pins', () => { + assert.deepEqual(naming(/(?:candidate|gpg|release)-fixture\.mjs/u), []); + assert.deepEqual(naming(/custody-fixtures/u), ['tools/cli-package/release-recipient.mjs']); +}); + +test('only the release gate imports encryptOnce', () => { + assert.deepEqual( + naming(/['"](?:\.\/|\.\.\/cli-package\/|tools\/cli-package\/)custody\.mjs['"]/u), + ['tools/cli-package/release-custody.mjs'], + ); +}); + +test('no product path or workflow names a release module yet', () => { + // Interim: Step 9d wires the release modules into the originating job and updates this control. + assert.deepEqual(naming(/release-(?:custody|binding|recipient)\.mjs/u), [ + 'tools/cli-package/release-binding.mjs', + 'tools/cli-package/release-custody.mjs', + ]); +}); + +/** Every specifier a module's import closure reaches, relative files followed. @param {string} entry */ +function closure(entry) { + const seen = new Set(); + const specifiers = new Set(); + const visit = (path) => { + if (seen.has(path)) return; + seen.add(path); + const source = readFileSync(path, 'utf8'); + for (const match of source.matchAll( + /^\s*import\s+(?:[^'";]*?\s+from\s+)?['"]([^'"]+)['"]|^\s*export\s+(?:\*|\{[^}]*\})\s+from\s+['"]([^'"]+)['"]|\bimport\(\s*['"]([^'"]+)['"]\s*\)/gmu, + )) { + const specifier = match[1] ?? match[2] ?? match[3]; + specifiers.add(specifier); + if (specifier.startsWith('.')) visit(join(dirname(path), specifier)); + } + }; + visit(join(here, entry)); + return { + files: [...seen].map((path) => relative(here, path).split('\\').join('/')).sort(), + specifiers: [...specifiers].sort(), + }; +} + +test("the binding receipt's closure starts no process, so reuse makes no GnuPG call", () => { + const { files, specifiers } = closure('release-binding.mjs'); + assert.deepEqual(files, [ + 'archive.mjs', + 'custody-error.mjs', + 'files.mjs', + 'release-binding.mjs', + 'release-recipient.mjs', + 'ustar.mjs', + ]); + for (const forbidden of ['node:child_process', 'child_process', 'node:worker_threads']) { + assert.equal(specifiers.includes(forbidden), false, forbidden); + } + // The scan reads the imports it claims to: the gate's closure reaches GnuPG. + const gate = closure('release-custody.mjs'); + for (const reached of ['custody.mjs', 'gnupg.mjs', 'process.mjs', 'container.mjs']) { + assert.ok(gate.files.includes(reached), reached); + } + assert.ok(gate.specifiers.includes('node:child_process')); +}); + +test('no production recipient is committed in Wave 3', () => { + // Interim: Wave 4b commits the recipient directory and updates this control. + assert.equal(existsSync(PRODUCTION_RECIPIENT_ROOT), false); + const listed = spawnSync( + 'git', + ['ls-files', '--', posix.join('tools/cli-package', 'release-recipient')], + { cwd: repository, encoding: 'utf8' }, + ); + assert.equal(listed.status, 0); + assert.equal(listed.stdout, ''); +}); + +test('the interim release guard still refuses a complete synthetic candidate under the receipt name', () => { + const directory = fresh('guarded-candidate'); + mkdirSync(directory); + writeFileSync(join(directory, RELEASE_BINDING_FILE), '{}'); + writeFileSync(join(directory, tarballName('0.1.1')), 'tarball'); + writeFileSync(join(directory, ciphertextFileName('0.1.1')), 'ciphertext'); + assert.throws( + () => requireReleaseCandidate(directory), + (error) => error instanceof ReleaseGuardError && error.code === 'RELEASE_CUSTODY_UNAVAILABLE', + ); +}); + +test('the constants the release modules restate equal their sources', () => { + for (const version of ['0.1.1', '1.2.3-rc.1', '2.0.0+build.5']) { + assert.equal(ciphertextFileName(version), ciphertextName(version)); + } + assert.equal(CIPHERTEXT_BYTES, CONTAINER_LIMITS.totalBytes + 64 * 1024); + assert.deepEqual(SYNTHETIC_SOURCE, { commit: COMMIT, tree: TREE }); +}); From d94ed705c4f53dc8353b8c7ec173769db6f7f91e Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 21:05:08 +0300 Subject: [PATCH 82/97] docs(repo): record the Step 9b.2 implementation packaging.md gains the Step 9b.2 boundary, the canonical home of the release binding receipt v1's schema, validator, sealer, reuse, loader and scope rules. deferred-tasks.md records three named limits (an orphaned ciphertext, no recipient re-check on reuse, Windows not exercised), the Step 9d handoffs and a Wave 4b clause on the committed recipient. The register records the implementation checkpoint with the generated and named mutation sweeps; the status lines name 9b.2 as implemented and under review. Refs: #46, ADR-0111 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 +- CLAUDE.md | 4 +- docs/reference/cli/README.md | 2 +- docs/reference/cli/packaging.md | 93 ++++++++++++++++++- docs/roadmap/current.md | 6 +- docs/roadmap/deferred-tasks.md | 42 ++++++++- .../phase-2.5.5-hardening-and-remediation.md | 46 ++++++++- tools/cli-package/README.md | 6 +- 8 files changed, 190 insertions(+), 13 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 362b9580..b2af541d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,7 +117,7 @@ explicit; source CI does not certify later documentation commits. The joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now -accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI and Step 9b.1 are accepted too, and Steps 9b.2–19 have not started) and whole-wave acceptance remains Step 19. +accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI and Step 9b.1 are accepted too, Step 9b.2 is implemented and under review, and Steps 9c–19 have not started) and whole-wave acceptance remains Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification @@ -130,7 +130,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25): round 1 reported no blocker, high or medium finding and no regression, with a clean security sitting; every product file and directory identity check now compares exact bigints, its low findings are recorded in deferred-tasks.md, and its Windows clause closes after three green advisory Windows runs ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)). Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b-preflight-decisions--2026-10-25)); Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is independently accepted on `development` at `147c2788` (2026-10-25): rounds 1–3 were not accepted (one, one and two mediums, each corrected), and round 4 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-acceptance--2026-10-25)). Step 9b.2 is next. `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25): round 1 reported no blocker, high or medium finding and no regression, with a clean security sitting; every product file and directory identity check now compares exact bigints, its low findings are recorded in deferred-tasks.md, and its Windows clause closes after three green advisory Windows runs ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)). Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b-preflight-decisions--2026-10-25)); Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is independently accepted on `development` at `147c2788` (2026-10-25): rounds 1–3 were not accepted (one, one and two mediums, each corrected), and round 4 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-acceptance--2026-10-25)). Step 9b.2 (the release binding receipt v1, verified ciphertext reuse, the production recipient loader, absent in Wave 3, and scope separation; synthetic recipients only) is implemented on `development` and under review ([decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b2-preflight-decisions--2026-10-25), [checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b2-implementation-checkpoint--2026-10-25)). `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. See [docs/roadmap/current.md](docs/roadmap/current.md) for live status. diff --git a/CLAUDE.md b/CLAUDE.md index 1c958286..94669cb1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -206,7 +206,7 @@ explicit; source CI does not certify later documentation commits. The joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now -accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI and Step 9b.1 are accepted too, and Steps 9b.2–19 have not started) and whole-wave acceptance remains Step 19. +accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI and Step 9b.1 are accepted too, Step 9b.2 is implemented and under review, and Steps 9c–19 have not started) and whole-wave acceptance remains Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification @@ -219,7 +219,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25): round 1 reported no blocker, high or medium finding and no regression, with a clean security sitting; every product file and directory identity check now compares exact bigints, its low findings are recorded in deferred-tasks.md, and its Windows clause closes after three green advisory Windows runs ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)). Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b-preflight-decisions--2026-10-25)); Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is independently accepted on `development` at `147c2788` (2026-10-25): rounds 1–3 were not accepted (one, one and two mediums, each corrected), and round 4 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-acceptance--2026-10-25)). Step 9b.2 is next. `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25): round 1 reported no blocker, high or medium finding and no regression, with a clean security sitting; every product file and directory identity check now compares exact bigints, its low findings are recorded in deferred-tasks.md, and its Windows clause closes after three green advisory Windows runs ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)). Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b-preflight-decisions--2026-10-25)); Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is independently accepted on `development` at `147c2788` (2026-10-25): rounds 1–3 were not accepted (one, one and two mediums, each corrected), and round 4 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-acceptance--2026-10-25)). Step 9b.2 (the release binding receipt v1, verified ciphertext reuse, the production recipient loader, absent in Wave 3, and scope separation; synthetic recipients only) is implemented on `development` and under review ([decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b2-preflight-decisions--2026-10-25), [checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b2-implementation-checkpoint--2026-10-25)). `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. For live status, per-PR history, milestone dates, and open obligations, see the canonical diff --git a/docs/reference/cli/README.md b/docs/reference/cli/README.md index a0ad0da1..0bc7fd30 100644 --- a/docs/reference/cli/README.md +++ b/docs/reference/cli/README.md @@ -13,5 +13,5 @@ Part of [reference/](../README.md). | [commands.md](commands.md) | The full `relavium` command surface (run, chat, chat-resume/list/export, agent run, list, create, import, export, logs, gate, init, provider) plus interactive TUI vs CI JSON mode. | | [home.md](home.md) | The bare-invocation interactive **Home** — the TTY gate, the read-only management strip over `history.db`, the Home↔chat mode machine + keymap, bracketed paste, the min-terminal degrade, and the signal/exit-code lifecycle (2.5.B / ADR-0054). | | [chat-session.md](chat-session.md) | The `relavium chat` agent-session REPL — entry, agent/model selection, the multi-turn loop, streaming, tool availability, the `--json` session-event stream, and exit code 4. | -| [packaging.md](packaging.md) | Wave 3 finite runtime graph, installed-package consumers and Step 8 native activation accepted on `development`; the Step 9a.1 candidate build and finalizer are accepted on `development` and Step 9a.2a's private manifest and symbolication are accepted on `development` and Step 9a.2b's product map controls and `package_test` receipt are accepted on `development`; Step 9b.1's private container, isolated GnuPG runner and encrypt-once library are accepted on `development` (synthetic recipients only); the remaining Step 9 custody and release gates remain staged. | +| [packaging.md](packaging.md) | Wave 3 finite runtime graph, installed-package consumers and Step 8 native activation accepted on `development`; the Step 9a.1 candidate build and finalizer are accepted on `development` and Step 9a.2a's private manifest and symbolication are accepted on `development` and Step 9a.2b's product map controls and `package_test` receipt are accepted on `development`; Step 9b.1's private container, isolated GnuPG runner and encrypt-once library are accepted on `development` (synthetic recipients only); Step 9b.2's ciphertext reuse, release binding receipt v1, absent production loader and scope separation are implemented on `development` and under review; the remaining Step 9 custody and release gates remain staged. | | [agent-run-fixture.md](agent-run-fixture.md) | The `relavium agent run --fixture` cassette format — a `StreamChunk[][]` JSON recording of an agent run's LLM stream, for deterministic offline replay. | diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 2d829bd9..23503486 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -1,6 +1,6 @@ # CLI Packaging Contract -- **Status**: Compiled graph, installed package and Step 8 native activation independently accepted on `development`; the Step 9a.1 candidate build and finalizer are independently accepted on `development` (2026-10-24); Step 9a.2a's strict decoder, private manifest and symbolication are independently accepted on `development` (2026-10-24); Step 9a.2b's product map controls, post-receipt rebuild refusals and `package_test` receipt are independently accepted on `development` (2026-10-24); Step 9b.1's private container, isolated GnuPG runner and encrypt-once library are independently accepted on `development` (2026-10-25; synthetic recipients only); the remaining Step 9 custody and release gates remain staged +- **Status**: Compiled graph, installed package and Step 8 native activation independently accepted on `development`; the Step 9a.1 candidate build and finalizer are independently accepted on `development` (2026-10-24); Step 9a.2a's strict decoder, private manifest and symbolication are independently accepted on `development` (2026-10-24); Step 9a.2b's product map controls, post-receipt rebuild refusals and `package_test` receipt are independently accepted on `development` (2026-10-24); Step 9b.1's private container, isolated GnuPG runner and encrypt-once library are independently accepted on `development` (2026-10-25; synthetic recipients only); Step 9b.2's ciphertext reuse, release binding receipt v1, absent production loader and scope separation are implemented on `development` and under review (synthetic recipients only); the remaining Step 9 custody and release gates remain staged - **Date**: 2026-10-15 - **Related**: [ADR-0111](../../decisions/0111-cli-meta-operations-use-a-lazy-packaged-runtime.md), [commands.md](commands.md), [release runbook](../../runbooks/release-a-surface.md) @@ -616,6 +616,97 @@ recorded in the [Wave 3 register](../../roadmap/phases/phase-2.5.5-hardening-and tools; `encryptOnce` itself refuses there. The 2.2.8 floor is enforced by the version check without being exercised. +### Recipient custody implementation boundary — Step 9b.2 + +Step 9b.2 implements the reuse of a retained ciphertext without re-encryption, the release binding +receipt's schema v1 with its validator and sealer, the production recipient loader (absent in Wave 3) +and the separation of synthetic and production scope +([decisions](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b2-preflight-decisions--2026-10-25)). +It is release tooling with synthetic controls only, in new modules: no Step 9b.1 module changes and +nothing outside its tests calls it. Step 9d wires it into the originating job, binds the receipt's +origin and map-check claims to the trusted run, and replaces the interim release guard, which still +refuses every artifact. Its review and acceptance are recorded in the +[Wave 3 register](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments). + +- **The production recipient.** `release-recipient.mjs` loads it from one committed directory, + `tools/cli-package/release-recipient/`, holding exactly `recipient.asc` (the + `gpg --armor --export '!'` export, at most 64 KiB) and `pins.json`: + `{"version": 1, "recipients": [{"primary", "subkey", "status"}]}` with 1–32 entries of 40-digit + uppercase fingerprints, each `current` or `retired`, exactly one `current` and no subkey twice. An + entry is never removed, so a candidate sealed before a rotation still verifies. There is no + environment variable, flag or fallback. In Wave 3 the directory does not exist and the loader + refuses `PRODUCTION_RECIPIENT_ABSENT`; a linked or malformed directory refuses + `PRODUCTION_RECIPIENT_INVALID`, and a history naming a key of the committed synthetic fixtures + refuses `FIXTURE_RECIPIENT`. The loaded value is frozen and branded: only that value counts as a + production recipient, never a copy. Wave 4b commits the files. +- **Scope.** A synthetic candidate is one whose private manifest records the fixtures' sentinel + source, commit `c`×40 and tree `b`×40. `encryptCandidate` admits production scope only with the + loaded recipient and a clean, non-synthetic candidate, and encrypts to the recipient's current + pins (`RECIPIENT_SOURCE`, `DIRTY_SOURCE`, `SYNTHETIC_SOURCE`). It admits synthetic scope only with a + synthetic candidate (`FIXTURE_SCOPE`), so neither a fixture key nor the per-run round-trip key can + encrypt a product staging. An unknown scope refuses `SCOPE`, and a destination already holding a + receipt refuses `ALREADY_SEALED`. Every one of these refuses before any GnuPG call; `encryptOnce` + then runs unchanged. +- **The release binding receipt v1.** The public `release-binding-receipt.json` beside the ciphertext, + the name the interim guard already keys on: + + | Field | Content | Checked here | Trust Step 9d adds | + | --- | --- | --- | --- | + | `version`, `kind` | `1`, `"release_binding"` | exact | — | + | `packageVersion` | the private manifest's version | equals the expectation | the tag | + | `source` | the manifest's `{commit, tree, dirty}` | the commit equals the expectation; production refuses `dirty` and the synthetic source, synthetic requires it | the run's commit | + | `tarball` | `{name, bytes, sha256}`, `relavium-.tgz`, 18 B–16 MiB | the name follows the version; the sealer binds the digest to the manifest's `tarballSha256` | — | + | `ciphertext` | `{name, bytes, sha256}`, `relavium--private-maps.tar.gpg`, 1 B–128 MiB + 64 KiB | the name follows the version; the sealer re-reads the file | — | + | `recipient` | `{primary, subkey}` | production: a pair of the loaded history; synthetic: the expected pins | — | + | `toolchain` | the manifest's node, pnpm, typescript, tsup and esbuild versions, and the GnuPG version | format | — | + | `target` | the manifest's target | format | — | + | `inputs` | the manifest's per-file lock and configuration digests, by path | format: 1–64 repository-relative paths | — | + | `mapCheck` | `{"status": "complete"}` | exact | a successful `ci` run with the product map controls at `source.commit` | + | `origin` | `{kind: "github", repository, workflow, runId, runAttempt, event: "push", source}` | format; `source` equals `source.commit` | the originating run and attempt | + + It carries no map, source, per-source inventory, host path or user ID, and no scope field: scope + always comes from the verifier's expectation. Every field is a claim until Step 9d binds the last + two. Release smoke on a reused candidate derives the public file inventory and package metadata from + the digest-verified tarball (Step 9d). +- **Canonical bytes.** The receipt is `JSON.stringify(receipt, null, 2)` plus a newline, its fields in + the table's order and its inputs sorted by path. A verifier parses the exact bytes, at most 64 KiB, + and refuses `BINDING_BYTES` for anything but the canonical serialization of the value it validates: + a duplicated key, a reordering, other whitespace, a BOM or trailing data. Verifiers never + re-serialize a receipt. +- **The validator.** `validateReleaseBinding(value, expected)` and `parseReleaseBinding(bytes, expected)` + take a required expectation: `{scope: 'production', recipient, packageVersion, source}` with the + loaded recipient, whose whole history verifies, or `{scope: 'synthetic', pins, packageVersion}`. + They refuse `BINDING_VERSION`, `BINDING_SHAPE` (an unknown, missing or malformed field at any level), + `BINDING_CANDIDATE`, `BINDING_NAME`, `BINDING_SIZE`, `BINDING_DIRTY`, `BINDING_SCOPE`, + `RECIPIENT_SOURCE`, `BINDING_RECIPIENT`, `BINDING_MAP_CHECK` and `BINDING_ORIGIN`. +- **The sealer.** `sealReleaseBinding(record, {origin})` accepts only a record `encryptCandidate` + returned and not yet sealed (`RECORD_SOURCE`); a failed seal leaves the record usable. It refuses an + existing receipt before reading anything (`ALREADY_SEALED`), admits exactly the tarball and the + ciphertext in the destination (`CANDIDATE_ENTRIES`), and re-reads the ciphertext's size and SHA-256 + against the record (`CIPHERTEXT_DRIFT`) and the tarball's digest against the manifest + (`TARBALL_DRIFT`). It validates its own receipt, then publishes it once: written to a fresh sibling, + then hard-linked to its name, which never replaces an existing receipt. A sibling that cannot be + removed after the link is left behind rather than reported, because the receipt is then published. + A ciphertext no record names, after a crash between encryption and sealing, can never be sealed and + needs manual recovery. +- **Reuse.** `verifyRetainedCandidate({directory, expected})` admits a directory holding exactly the + receipt, the tarball and the ciphertext (`CANDIDATE_ENTRIES`) and parses the receipt. It checks each + asset's size (`CANDIDATE_SIZE`), reads it as a bounded ordinary file (`CANDIDATE_ASSET`), checks its + SHA-256 (`CANDIDATE_DIGEST`), and returns the verified bytes, never a path to open again. It writes, + renames and removes nothing. Its module imports nothing that can start a process, so reuse makes no + GnuPG call, and it cannot re-check the ciphertext's recipients: a named limit. +- **Diagnostics.** Every refusal is a fixed `PACKED_CLI_CUSTODY_`, a host failure `UNEXPECTED`; no + path, key or receipt content surfaces. +- **Where it runs.** `release-recipient.test.mjs`, `release-binding.test.mjs`, `release-custody.test.mjs` + and `release-scope.test.mjs` join `test:acceptance-tools`, as the 9b.1 tests do; the custody test + needs GnuPG ≥ 2.2.8. `release-scope.test.mjs` pins the separation statically: + - no product or tooling code reaches a test fixture, and only the loader reads the fixture pins; + - only the release gate imports `encryptOnce`; + - the receipt module's import closure starts no process; + - the interim guard still refuses a complete synthetic candidate; + - until Step 9d no product path or workflow names a release module, and until Wave 4b no recipient + directory is committed. + ## Package-test and release modes **Package-test mode** is required on PR/push and manual dry-run. In its isolated diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index 45adfc0e..cc0c7b01 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -138,7 +138,7 @@ The [deferred register](deferred-tasks.md) retains its existing limits; closure **Wave 3 Steps 1–8 merged to `main` on 2026-10-18 as [PR #95](https://github.com/HodeTech/Relavium/pull/95), merge commit `6d2a7372` (22:44 UTC). Wave 3 itself is not complete: Steps 9–19 had not started at -that merge (Steps 9a.1, 9a.2a and 9a.2b are now accepted on `development`, the native-identity increment NI and Step 9b.1 are accepted too, and Steps 9b.2–19 have not started) and whole-wave acceptance +that merge (Steps 9a.1, 9a.2a and 9a.2b are now accepted on `development`, the native-identity increment NI and Step 9b.1 are accepted too, Step 9b.2 is implemented and under review, and Steps 9c–19 have not started) and whole-wave acceptance remains Step 19.** The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) joins the completed scope, every review attribution, the independent review's dispositions and the publication state observed at merge. It was the maintainer-authorized, scoped publication checkpoint. @@ -172,7 +172,7 @@ bar and round cap, as its medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, by the author alone). Its other findings are recorded in -[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is independently accepted on `development` at `cf97b7f1` (2026-10-24): its round 13 reported no blocker, high or medium finding and its security sitting was clean, and the round's low findings are recorded in deferred-tasks.md ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)); Step 9a.2 is split into 9a.2a and 9a.2b, and Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24) after three rounds, the third with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)); Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24) after two rounds, the second with no blocker, high or medium finding and no regression ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)), and the native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25) after one round with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)), its Windows clause open until three green advisory Windows runs; Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b-preflight-decisions--2026-10-25)), and Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is independently accepted on `development` at `147c2788` ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-acceptance--2026-10-25)), with Step 9b.2 next; the substrate's +[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is independently accepted on `development` at `cf97b7f1` (2026-10-24): its round 13 reported no blocker, high or medium finding and its security sitting was clean, and the round's low findings are recorded in deferred-tasks.md ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)); Step 9a.2 is split into 9a.2a and 9a.2b, and Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24) after three rounds, the third with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)); Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24) after two rounds, the second with no blocker, high or medium finding and no regression ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)), and the native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25) after one round with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)), its Windows clause open until three green advisory Windows runs; Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b-preflight-decisions--2026-10-25)), and Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is independently accepted on `development` at `147c2788` ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-acceptance--2026-10-25)), and Step 9b.2 (the release binding receipt v1, verified ciphertext reuse, the absent production recipient loader and scope separation) is implemented on `development` and under review ([checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b2-implementation-checkpoint--2026-10-25)); the substrate's 9a.1–9d increments carry a dedicated archive/crypto security sitting. Steps 10–19 follow the [increment table](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) and end with whole-wave acceptance at Step 19. `development` is pushed after each accepted increment; PR #95 no longer bounds the @@ -1141,7 +1141,7 @@ an ESLint selector rather than a list. ### Wave 3 — Clear the ground -**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Steps 9a.1, 9a.2a, 9a.2b, NI and 9b.1 accepted on `development` (2026-10-24/25), Steps 9b.2–19 NOT STARTED — 2026-10-25.** +**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Steps 9a.1, 9a.2a, 9a.2b, NI and 9b.1 accepted on `development` (2026-10-24/25), Step 9b.2 implemented and under review, Steps 9c–19 NOT STARTED — 2026-10-25.** ADR-0110–0113, their linked contracts and the nineteen-increment plan were approved on 2026-10-16. [Steps 1–8 were independently accepted](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) on `development`, with whole Step 8 reviewed at `c83c8575`, and published as the scoped checkpoint diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index 848586c3..5e74ef9f 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -2,7 +2,7 @@ > Status: Living -> Last updated: 2026-10-25 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Step 9a.1 is independently accepted on `development` with its recorded low findings below, Step 9a.2a is independently accepted on `development` with its recorded low findings below, Step 9a.2b is independently accepted on `development` with its recorded low findings below, the native-identity increment NI independently accepted on `development` with its recorded low findings below (its Windows clause open), Step 9b.1 independently accepted on `development` with its named limits and recorded low findings below, and Steps 9b.2–19 not started; the PR95 increment is closed on `development` under the maintainer's 2026-10-21 round cap). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's edge cases with the maintainer's 2026-10-21 acceptance bar and round cap and the 2026-10-22 rulings. Existing approved deferrals remain. +> Last updated: 2026-10-25 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Step 9a.1 is independently accepted on `development` with its recorded low findings below, Step 9a.2a is independently accepted on `development` with its recorded low findings below, Step 9a.2b is independently accepted on `development` with its recorded low findings below, the native-identity increment NI independently accepted on `development` with its recorded low findings below (its Windows clause open), Step 9b.1 independently accepted on `development` with its named limits and recorded low findings below, Step 9b.2 implemented and under review with its named limits and handoffs below, and Steps 9c–19 not started; the PR95 increment is closed on `development` under the maintainer's 2026-10-21 round cap). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's edge cases with the maintainer's 2026-10-21 acceptance bar and round cap and the 2026-10-22 rulings. Existing approved deferrals remain. > The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) records completed prerequisites; full M/N lifecycle and other scheduled work remain with their owners. > > Earlier triage: the Phase-2.6 rewrite triaged every open item, the 2026-07-19 full-project @@ -863,6 +863,40 @@ Recorded with the Step 9b.1 acceptance after round 4 directories of the job, or compare against the mount table, before relying on the refusal; narrow the comment on the next change to `custody.mjs`. *(low · tools/cli-package/custody.mjs)* +## Wave 3 Step 9b.2 — named limits and handoffs (2026-10-25) + +Recorded with the Step 9b.2 implementation +([decisions](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b2-preflight-decisions--2026-10-25)). +The release modules are library code with synthetic controls only: nothing outside their tests calls them. +**Trigger for every entry:** Step 9d's release integration, or the next change to the file it names. + +- [ ] **An orphaned ciphertext can never be sealed.** *Low; a named limit.* A crash after `encryptCandidate` + publishes the ciphertext and before `sealReleaseBinding` runs leaves a ciphertext that no record names, and + only a record the gate returned in the same process seals. The candidate then needs manual recovery + (packaging.md's missing-asset state); it is never re-encrypted. **Re-check:** in Step 9c's reservation + states, classify such a directory as incomplete and route it to manual recovery. *(low · + tools/cli-package/release-custody.mjs)* +- [ ] **Reuse cannot re-check the ciphertext's recipients.** *Low; a named limit.* `verifyRetainedCandidate` + binds each asset's size and SHA-256 to the receipt, and the receipt's recipient to the expected history, but + re-reading the ciphertext's public-key packets (`ENC_TO`) needs GnuPG, which reuse never calls. The digest + ties the bytes to the ones the sealer re-read after encryption checked their single recipient. **Re-check:** + if a retained ciphertext can ever come from anything but the sealer, verify its recipients with the isolated + runner before reuse. *(low · tools/cli-package/release-binding.mjs)* +- [ ] **The release modules do not run on Windows.** *Low; a named limit.* Like the 9b.1 tests, the release + tests run only in `test:acceptance-tools`, which no Windows job runs; `encryptCandidate` refuses there + through `encryptOnce`'s `PRIVATE_ROOT`, while reuse is platform-neutral but unexercised. **Re-check:** with + the 9b.1 Windows limit. *(low · tools/cli-package/release-binding.mjs)* +- [ ] **Step 9d handoffs.** *Scheduled → Step 9d.* Wire `encryptCandidate` and `sealReleaseBinding` into the + originating job, with the origin from `originFromEnvironment`; bind the receipt's `origin` to the trusted run + and attempt (on reuse, the original run) and `mapCheck` to a successful `ci` run with the product map + controls at `source.commit`; derive release smoke's public inventory and package metadata from the + digest-verified tarball with the package-test inspector; check the production recipient before any + reservation, so a recipient refusal never leaves an incomplete reservation; widen the sealer's directory + admission if the package-test receipt travels beside the tarball; replace the interim guard; then update + `release-scope.test.mjs`'s interim control that no product path or workflow names a release module. The + guard test's non-canonical `relavium-0.1.1.tgz.gpg` name is corrected with the guard. *(scheduled · + tools/cli-package/release-custody.mjs, tools/cli-package/release-guard-core.mjs)* + ## Wave 3 NI — recorded low findings (2026-10-25) Round 1 of the native-identity increment at `fc5af701` accepted it with these lows @@ -2743,7 +2777,11 @@ future test cannot silently re-acquire it. primary with only the pinned encryption subkey (`gpg --armor --export '!'`), because a second usable encryption subkey refuses. *Added after Step 9b.1's review round 1:* its cipher preferences must include AES256, since custody passes an AES-256 preference and refuses any - other cipher. + other cipher. *Added with Step 9b.2:* the recipient is committed at + `tools/cli-package/release-recipient/` as `recipient.asc` and `pins.json`, whose history is append-only + (exactly one `current`, a rotated recipient `retired`, never removed); a fixture fingerprint refuses. Wave 4b + then updates `release-recipient.test.mjs`'s absence test and `release-scope.test.mjs`'s interim control that no + recipient directory is committed. Any expiry margin is passed as the `now` the custody check uses. - [ ] **Scheduled → Wave 5a: alt-screen word wrapping (#61).** The maintainer approved retaining character-wrap replacement with render-v2/D18 and picker diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index ba8e87fa..ba9e4df9 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -3704,7 +3704,7 @@ No credential values or request captures belong in tracked evidence. | NI | **Native file identity on Windows (maintainer-approved 2026-10-24; after 9a.2b, before 9b):** every file and directory identity check compares `dev`/`ino` read with `{ bigint: true }` — the filesystem policy's pinned roots, the workflow catalog's open check and the media store's pins, read check and temporary claims — so a replaced file or directory can never compare equal on Windows, where `ino` is a 64-bit NTFS file ID that a JavaScript number cannot hold exactly; a replacement test per site. Found by 9a.2a's push CI ([acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)); recorded in [deferred-tasks.md](../deferred-tasks.md#native-file-identity-on-windows--approved-increment-2026-10-24). Security sitting. *Preflight decisions approved on 2026-10-25 ([decisions](#wave-3-ni-preflight-decisions--2026-10-25)); independently accepted on `development` at `fc5af701` on 2026-10-25 ([checkpoint](#wave-3-ni-implementation-checkpoint--2026-10-25), [acceptance](#wave-3-ni-acceptance--2026-10-25)); its Windows clause closes after three green advisory Windows runs on pushed heads.* | | 9b | **Recorded split — recipient custody:** synthetic-only isolated GnuPG controls (public-only home/keyring, pinned primary and encryption-subkey fingerprints, machine-readable refusal of missing, expired, revoked or unusable recipients), encrypt once and reuse the original ciphertext. Explicit crypto/custody security sitting. Fixture proof never claims production recipient or custody configuration. *Split on 2026-10-25 (maintainer-approved, L8) into 9b.1 and 9b.2 below, which replace it as increments ([preflight decisions](#wave-3-step-9b-preflight-decisions--2026-10-25)).* | | 9b.1 | **Recorded split, 2026-10-25 (maintainer-approved) — container, isolated GnuPG runner and encrypt-once:** the strict USTAR dialect extracted from the archive reader with its writer promoted; the private container built from the sealed staging in a staging-relative layout, each map's digest checked against the private manifest and its size against the finalizer's record, which must name exactly the manifest's outputs and maps (the metafile must be a JSON object); an isolated public-only GnuPG runner (GnuPG ≥ 2.2.8; `--recipient-file` with no keyring or agent, and no trustdb but the empty one GnuPG may create in the listing home); a colon-listing pre-check that refuses missing, expired, revoked, unusable (v4 with cv25519 or RSA ≥ 3072 only) or unexpected recipients and secret material; `encryptOnce` (SEIPDv1 with MDC and AES-256, exactly one `ENC_TO` for the pinned subkey, the exact produced asset, a no-clobber publish, plaintext cleanup on every path); committed synthetic public fixtures, an ephemeral round-trip key, and the GnuPG prerequisite documented. Library and synthetic controls only: product paths and workflows stay unchanged. Crypto/custody security sitting. *Independently accepted on `development` at `147c2788` (2026-10-25) ([decisions](#wave-3-step-9b-preflight-decisions--2026-10-25), [checkpoint](#wave-3-step-9b1-implementation-checkpoint--2026-10-25), [acceptance](#wave-3-step-9b1-acceptance--2026-10-25)).* | -| 9b.2 | **Recorded split, 2026-10-25 (maintainer-approved) — reuse, binding receipt and scope separation:** retained-ciphertext reuse without re-encryption (digest and size verified, zero GnuPG calls); the release binding receipt's schema v1, its validator and a sealer that writes only after the ciphertext exists (9d adds the origin, map-check and `ci`-run binding); the production recipient loader, absent in Wave 3; synthetic and production scope separation. Crypto/custody security sitting. | +| 9b.2 | **Recorded split, 2026-10-25 (maintainer-approved) — reuse, binding receipt and scope separation:** retained-ciphertext reuse without re-encryption (digest and size verified, zero GnuPG calls); the release binding receipt's schema v1, its validator and a sealer that writes only after the ciphertext exists (9d adds the origin, map-check and `ci`-run binding); the production recipient loader, absent in Wave 3; synthetic and production scope separation. Crypto/custody security sitting. *Implemented on `development` and under review ([decisions](#wave-3-step-9b2-preflight-decisions--2026-10-25), [checkpoint](#wave-3-step-9b2-implementation-checkpoint--2026-10-25)).* | | 9c | **Recorded split — offline release state controller:** permission-enforcing offline GitHub/npm controller, permissioned prebuild discovery and reservation, and every rerun, partial, conflict and expiry state, with no overwrite, rebuild or re-encryption on uncertain history. | | 9d | **Recorded split — workflow, runbook and whole-Step-9 landing:** least-privilege actual workflow dependency graph, required artifact/platform aggregation and exact-artifact consumers, lost-staging refusal and the release runbook update. The whole-Step-9 cumulative review retains all 9a–9d changes with preservation traces into Step 8, and the dedicated archive/crypto security sitting closes the step. | | 10 | **CLI behaviour/discovery integration:** #45/#264/#265/#268/#13/#17/#18/#310/#309/#19 under ADR-0110, consuming Steps 6–9. Production owner category matrix, real root/subtree JSON help, pricing/chat aliases, non-echoing Commander refusal and signal-safe deletion consent. G15 is refuted and receives real engine/store/log preservation evidence, with no invented token coalescing. Repeat affected packed gates. | @@ -10377,3 +10377,47 @@ Engineering defaults recorded with the rulings: - **Gate.** A rigor pass and a generated mutation sweep before the first round; the full `pnpm run ci` alone; the custody tests on Ubuntu 24.04 with GnuPG 2.4.4 in Docker; one new complete cumulative round and a separate crypto/custody security sitting. + +### Wave 3 Step 9b.2 implementation checkpoint — 2026-10-25 + +Step 9b.2 is implemented on `development` in `84b8e455..4d49176b` and this record, under the +[preflight decisions](#wave-3-step-9b2-preflight-decisions--2026-10-25). Its canonical description is packaging.md's +[Step 9b.2 boundary](../../reference/cli/packaging.md#recipient-custody-implementation-boundary--step-9b2). No Step +9b.1 module changed, and nothing outside the new tests calls the new modules. +- **The production recipient** (`release-recipient.mjs`). The loader reads the committed + `tools/cli-package/release-recipient/` directory and refuses `PRODUCTION_RECIPIENT_ABSENT` in Wave 3, because no + recipient is committed. A linked or malformed directory refuses `PRODUCTION_RECIPIENT_INVALID`, and a history naming + a committed fixture key refuses `FIXTURE_RECIPIENT`. The loaded value is branded through a module-private set. +- **The receipt, its validator and reuse** (`release-binding.mjs`). Schema v1 carries every field the contract lists, + with a required expectation and canonical bytes. Reuse admits exactly the three files and returns the verified bytes. + The module's import closure, pinned by a test, holds only `archive.mjs`, `custody-error.mjs`, `files.mjs`, + `release-recipient.mjs` and `ustar.mjs`. +- **The gate and the sealer** (`release-custody.mjs`). Every scope refusal comes before any GnuPG call, and a + module-private registry ties the sealer to the records the gate returned. The receipt is published once through a + hard link, and a sibling left after the link is not a refusal. +- **Tests.** Four new files with 34 tests join `test:acceptance-tools`: `release-recipient`, `release-binding`, + `release-custody` and `release-scope`. They include a production round trip, in which an ephemeral key under a + rotated history decrypts to the container whose manifest the receipt binds, and fault injection after the link and + after encryption. +- **Docs.** packaging.md's Step 9b.2 boundary is the schema's canonical home. deferred-tasks.md records three named + limits (an orphaned ciphertext, no recipient re-check on reuse, Windows not exercised), the Step 9d handoffs and a + Wave 4b clause on the recipient's committed form. + +**Sweeps.** A generated sweep mutated every one-line refusal, condition clause and comparison of the three modules: +131 mutants, 119 killed at first. Five survivors were real gaps and gained cases: +- a `pending` status beside the current one; +- a one-entry inputs map; +- a receipt passed as a `Uint8Array`; +- a host failure in the gate, and one after encryption. + +The other seven are equivalent: +- the history's type and read checks: removing one leads to a host error that the loader's wrapper maps to the same + code, or for an empty history to the one-current rule; +- the inputs sort comparator, whose keys are unique; +- the receipt bound's `>=` flip, since no canonical receipt reaches 64 KiB. + +24 named semantic mutants, which the generator cannot produce, all fail a test. They cover the brand, the fixture +denylist, the root's shape, the current pins, the registry's consumption, the best-effort removal, the `EEXIST` +mapping, the canonical bytes, the sealer's fields and the link-aware presence check. + +A rigor pass, the full CI and the Ubuntu 2.4.4 evidence follow before the first round. diff --git a/tools/cli-package/README.md b/tools/cli-package/README.md index 72f48f10..5a1ac60a 100644 --- a/tools/cli-package/README.md +++ b/tools/cli-package/README.md @@ -85,7 +85,11 @@ Windows): a colon pre-check refuses missing, expired, revoked, unusable or unexp secret material it can detect, the ciphertext must be SEIPDv1 with MDC and AES-256 naming exactly the pinned subkey, an existing ciphertext is never overwritten, the plaintext work directory is removed on every path and every refusal is a fixed `PACKED_CLI_CUSTODY_*` code. -`custody-fixtures/` holds the committed synthetic public keys; no secret key is committed. No private map, +`custody-fixtures/` holds the committed synthetic public keys; no secret key is committed. Step 9b.2 adds, again +with nothing calling it yet, the release binding receipt v1 with its validator and verified reuse +(`release-binding.mjs`, whose import closure starts no process), the scope gate and the sealer that writes the +receipt once after re-reading the ciphertext (`release-custody.mjs`), and the production recipient loader +(`release-recipient.mjs`), which refuses in Wave 3 because no recipient is committed. No private map, metafile, manifest or per-source inventory enters the public artifact. The release workflow's `release-guard.mjs` refuses every artifact until verified release custody exists: it sets its exit code to 1 before it evaluates its rule (`release-guard-core.mjs`), so a rule From 91d08a657be0d0433fa77b803af78b35416ad8ce Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 25 Oct 2026 22:13:54 +0300 Subject: [PATCH 83/97] fix(repo): close the Step 9b.2 internal rigor pass Mediums corrected: - the loader's exported root and fixtures parameters let any caller brand any key, a fixture or the per-run key included; the loader now takes no argument, tests use a separately named seam, and permanent controls keep the loader argument-free and the seam out of product and tooling code; - the encryptOnce import control matched three spellings; it now resolves every quoted specifier against its file and self-tests; - the primary half of the recipient pair had no test. Lows corrected: the gate reads its scope once and encrypts its own copy of the container, and the validator snapshots its expectation; the gate resolves the destination (PLACEMENT) and checks the tarball (TARBALL_DRIFT) before encrypting, so nothing after publication can fail on the destination; integer-like input names refuse; the fixtures' pins file is parsed strictly; the closure control reads export * as, forbids cluster and module and scans for require-like calls; the guard control uses a valid synthetic candidate; new tests and wording. Four lows are recorded in deferred-tasks.md. Refs: #46, ADR-0111 Co-Authored-By: Claude Opus 5.5 --- docs/reference/cli/packaging.md | 58 +++++--- docs/roadmap/deferred-tasks.md | 32 ++++- .../phase-2.5.5-hardening-and-remediation.md | 48 ++++++- tools/cli-package/README.md | 4 +- tools/cli-package/release-binding.mjs | 64 ++++++--- tools/cli-package/release-binding.test.mjs | 95 +++++++++++-- tools/cli-package/release-custody.mjs | 81 ++++++----- tools/cli-package/release-custody.test.mjs | 109 +++++++++++++-- tools/cli-package/release-recipient.mjs | 48 +++++-- tools/cli-package/release-recipient.test.mjs | 45 ++++-- tools/cli-package/release-scope.test.mjs | 128 ++++++++++++++++-- 11 files changed, 572 insertions(+), 140 deletions(-) diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 23503486..281db75a 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -633,20 +633,28 @@ refuses every artifact. Its review and acceptance are recorded in the `gpg --armor --export '!'` export, at most 64 KiB) and `pins.json`: `{"version": 1, "recipients": [{"primary", "subkey", "status"}]}` with 1–32 entries of 40-digit uppercase fingerprints, each `current` or `retired`, exactly one `current` and no subkey twice. An - entry is never removed, so a candidate sealed before a rotation still verifies. There is no - environment variable, flag or fallback. In Wave 3 the directory does not exist and the loader - refuses `PRODUCTION_RECIPIENT_ABSENT`; a linked or malformed directory refuses - `PRODUCTION_RECIPIENT_INVALID`, and a history naming a key of the committed synthetic fixtures - refuses `FIXTURE_RECIPIENT`. The loaded value is frozen and branded: only that value counts as a - production recipient, never a copy. Wave 4b commits the files. + entry must never be removed, so that a candidate sealed before a rotation still verifies; the + loader sees one revision and cannot detect a removal, so that rule is kept by review until a + control over the file's history exists (a recorded Wave 4b handoff). `loadProductionRecipient()` + takes no argument: there is no environment variable, flag or fallback. In Wave 3 the directory + does not exist and the loader refuses `PRODUCTION_RECIPIENT_ABSENT`; a linked or malformed + directory, and any host failure while loading, refuses `PRODUCTION_RECIPIENT_INVALID`. A history + naming a fingerprint that the committed fixtures' `pins.json` lists refuses `FIXTURE_RECIPIENT`, and + a malformed fixtures file refuses rather than weaken that denylist. The loaded value is frozen and + branded: only that value counts as a production recipient, never a copy. Tests load temporary + roots through the separately named `loadRecipientForTests`, which `release-scope.test.mjs` + permanently keeps out of every product and tooling file. Wave 4b commits the files. - **Scope.** A synthetic candidate is one whose private manifest records the fixtures' sentinel source, commit `c`×40 and tree `b`×40. `encryptCandidate` admits production scope only with the loaded recipient and a clean, non-synthetic candidate, and encrypts to the recipient's current pins (`RECIPIENT_SOURCE`, `DIRTY_SOURCE`, `SYNTHETIC_SOURCE`). It admits synthetic scope only with a synthetic candidate (`FIXTURE_SCOPE`), so neither a fixture key nor the per-run round-trip key can - encrypt a product staging. An unknown scope refuses `SCOPE`, and a destination already holding a - receipt refuses `ALREADY_SEALED`. Every one of these refuses before any GnuPG call; `encryptOnce` - then runs unchanged. + encrypt a product staging. An unknown scope refuses `SCOPE`, a destination already holding a + receipt refuses `ALREADY_SEALED`, a destination that does not resolve refuses `PLACEMENT`, and a + destination without the tarball the manifest names refuses `TARBALL_DRIFT`, so a ciphertext is + never encrypted where it could not be sealed. Every one of these refuses before any GnuPG call. + The gate reads the caller's scope once and encrypts its own copy of the container, so neither can + change midway; `encryptOnce` then runs unchanged on that copy. - **The release binding receipt v1.** The public `release-binding-receipt.json` beside the ciphertext, the name the interim guard already keys on: @@ -669,14 +677,15 @@ refuses every artifact. Its review and acceptance are recorded in the two. Release smoke on a reused candidate derives the public file inventory and package metadata from the digest-verified tarball (Step 9d). - **Canonical bytes.** The receipt is `JSON.stringify(receipt, null, 2)` plus a newline, its fields in - the table's order and its inputs sorted by path. A verifier parses the exact bytes, at most 64 KiB, - and refuses `BINDING_BYTES` for anything but the canonical serialization of the value it validates: - a duplicated key, a reordering, other whitespace, a BOM or trailing data. Verifiers never - re-serialize a receipt. + the table's order and its inputs sorted by path; an integer-like input name, which JavaScript would + order first, refuses. A verifier parses the exact bytes, at most 64 KiB, and refuses + `BINDING_BYTES` for anything but the canonical serialization of the value it validates: a + duplicated key, a reordering, other whitespace, a BOM or trailing data. A verifier accepts only + those exact bytes and never writes a receipt back. - **The validator.** `validateReleaseBinding(value, expected)` and `parseReleaseBinding(bytes, expected)` take a required expectation: `{scope: 'production', recipient, packageVersion, source}` with the loaded recipient, whose whole history verifies, or `{scope: 'synthetic', pins, packageVersion}`. - They refuse `BINDING_VERSION`, `BINDING_SHAPE` (an unknown, missing or malformed field at any level), + Each reads the expectation once. They refuse `BINDING_VERSION`, `BINDING_SHAPE` (an unknown, missing or malformed field at any level), `BINDING_CANDIDATE`, `BINDING_NAME`, `BINDING_SIZE`, `BINDING_DIRTY`, `BINDING_SCOPE`, `RECIPIENT_SOURCE`, `BINDING_RECIPIENT`, `BINDING_MAP_CHECK` and `BINDING_ORIGIN`. - **The sealer.** `sealReleaseBinding(record, {origin})` accepts only a record `encryptCandidate` @@ -686,24 +695,29 @@ refuses every artifact. Its review and acceptance are recorded in the against the record (`CIPHERTEXT_DRIFT`) and the tarball's digest against the manifest (`TARBALL_DRIFT`). It validates its own receipt, then publishes it once: written to a fresh sibling, then hard-linked to its name, which never replaces an existing receipt. A sibling that cannot be - removed after the link is left behind rather than reported, because the receipt is then published. - A ciphertext no record names, after a crash between encryption and sealing, can never be sealed and - needs manual recovery. + removed after the link is left behind rather than reported, because the receipt is then published; + the candidate is not reusable (`CANDIDATE_ENTRIES`) until that sibling is removed by hand. A + ciphertext whose seal never succeeds before its process exits — after a crash, or a refusal on an + input the gate does not check, such as the origin — can never be sealed and needs manual recovery. - **Reuse.** `verifyRetainedCandidate({directory, expected})` admits a directory holding exactly the receipt, the tarball and the ciphertext (`CANDIDATE_ENTRIES`) and parses the receipt. It checks each asset's size (`CANDIDATE_SIZE`), reads it as a bounded ordinary file (`CANDIDATE_ASSET`), checks its SHA-256 (`CANDIDATE_DIGEST`), and returns the verified bytes, never a path to open again. It writes, renames and removes nothing. Its module imports nothing that can start a process, so reuse makes no GnuPG call, and it cannot re-check the ciphertext's recipients: a named limit. -- **Diagnostics.** Every refusal is a fixed `PACKED_CLI_CUSTODY_`, a host failure `UNEXPECTED`; no - path, key or receipt content surfaces. +- **Diagnostics.** Every refusal is a fixed `PACKED_CLI_CUSTODY_`. A host failure is `UNEXPECTED`, + except while loading the recipient (`PRODUCTION_RECIPIENT_INVALID`) or reading a named file, which + carries that file's code (`CANDIDATE_ASSET`, `CIPHERTEXT_DRIFT`, `TARBALL_DRIFT`). No path, key or + receipt content surfaces. - **Where it runs.** `release-recipient.test.mjs`, `release-binding.test.mjs`, `release-custody.test.mjs` and `release-scope.test.mjs` join `test:acceptance-tools`, as the 9b.1 tests do; the custody test needs GnuPG ≥ 2.2.8. `release-scope.test.mjs` pins the separation statically: - no product or tooling code reaches a test fixture, and only the loader reads the fixture pins; - - only the release gate imports `encryptOnce`; - - the receipt module's import closure starts no process; - - the interim guard still refuses a complete synthetic candidate; + - only the release gate imports `encryptOnce`, by any specifier that resolves to it; + - the loader is never called with an argument, and its test seam appears in no product or tooling + file (permanent controls); + - the receipt module's import closure starts no process, by import or otherwise; + - the interim guard still refuses a complete, valid synthetic candidate; - until Step 9d no product path or workflow names a release module, and until Wave 4b no recipient directory is committed. diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index 5e74ef9f..d862ae5f 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -870,9 +870,11 @@ Recorded with the Step 9b.2 implementation The release modules are library code with synthetic controls only: nothing outside their tests calls them. **Trigger for every entry:** Step 9d's release integration, or the next change to the file it names. -- [ ] **An orphaned ciphertext can never be sealed.** *Low; a named limit.* A crash after `encryptCandidate` - publishes the ciphertext and before `sealReleaseBinding` runs leaves a ciphertext that no record names, and - only a record the gate returned in the same process seals. The candidate then needs manual recovery +- [ ] **An orphaned ciphertext can never be sealed.** *Low; a named limit.* Only a record the gate returned in + the same process seals, so a ciphertext whose seal never succeeds before its process exits — after a crash, or + a refusal on an input the gate does not check, such as a malformed origin or an entry added to the + destination — is named by no record. The gate refuses an unresolvable destination and a missing or different + tarball before encrypting, which removes the likeliest causes. The candidate then needs manual recovery (packaging.md's missing-asset state); it is never re-encrypted. **Re-check:** in Step 9c's reservation states, classify such a directory as incomplete and route it to manual recovery. *(low · tools/cli-package/release-custody.mjs)* @@ -882,6 +884,18 @@ The release modules are library code with synthetic controls only: nothing outsi ties the bytes to the ones the sealer re-read after encryption checked their single recipient. **Re-check:** if a retained ciphertext can ever come from anything but the sealer, verify its recipients with the isolated runner before reuse. *(low · tools/cli-package/release-binding.mjs)* +- [ ] **A sibling left after the receipt's link keeps the candidate from reuse.** *Low; a named limit (rigor + pass, SIT-9B2-6).* When the sealer cannot remove its `.partial` sibling after the link, the receipt is + published and the seal succeeds, but `verifyRetainedCandidate` admits exactly three entries and refuses + `CANDIDATE_ENTRIES` until the sibling is removed by hand; a test pins that state. **Re-check:** in Step 9c's + state classification, report such a directory as sealed with a stray sibling. *(low · + tools/cli-package/release-custody.mjs)* +- [ ] **The fixture denylist covers only the fingerprints the fixtures' pins file lists.** *Low; a named limit + (rigor pass, CLAIM-9B2-2).* `two-encryption-subkeys.asc` carries a second encryption subkey, + `7BC68AC7D2944C2AB3240A521D784594073951D5`, that `custody-fixtures/pins.json` does not list, so a production + history naming it would load. The loader reads no key material (it makes no GnuPG call), and encrypting to it + would still need its armored key in the committed recipient file. **Re-check:** list every committed fixture + fingerprint in the denylist when the fixtures are next generated. *(low · tools/cli-package/release-recipient.mjs)* - [ ] **The release modules do not run on Windows.** *Low; a named limit.* Like the 9b.1 tests, the release tests run only in `test:acceptance-tools`, which no Windows job runs; `encryptCandidate` refuses there through `encryptOnce`'s `PRIVATE_ROOT`, while reuse is platform-neutral but unexercised. **Re-check:** with @@ -893,8 +907,11 @@ The release modules are library code with synthetic controls only: nothing outsi digest-verified tarball with the package-test inspector; check the production recipient before any reservation, so a recipient refusal never leaves an incomplete reservation; widen the sealer's directory admission if the package-test receipt travels beside the tarball; replace the interim guard; then update - `release-scope.test.mjs`'s interim control that no product path or workflow names a release module. The - guard test's non-canonical `relavium-0.1.1.tgz.gpg` name is corrected with the guard. *(scheduled · + `release-scope.test.mjs`'s interim control that no product path or workflow names a release module, keeping + its permanent controls (the loader takes no argument and its test seam stays out of product code; only the + gate imports `encryptOnce`); and include `tools/cli-package/custody-fixtures/pins.json` in every checkout + that loads the production recipient, because the loader reads it as its denylist. The guard test's + non-canonical `relavium-0.1.1.tgz.gpg` name is corrected with the guard. *(scheduled · tools/cli-package/release-custody.mjs, tools/cli-package/release-guard-core.mjs)* ## Wave 3 NI — recorded low findings (2026-10-25) @@ -2781,7 +2798,10 @@ future test cannot silently re-acquire it. `tools/cli-package/release-recipient/` as `recipient.asc` and `pins.json`, whose history is append-only (exactly one `current`, a rotated recipient `retired`, never removed); a fixture fingerprint refuses. Wave 4b then updates `release-recipient.test.mjs`'s absence test and `release-scope.test.mjs`'s interim control that no - recipient directory is committed. Any expiry margin is passed as the `now` the custody check uses. + recipient directory is committed. Any expiry margin is passed as the `now` the custody check uses. The + loader sees one revision of `pins.json` and cannot detect a removed or rewritten entry, so Wave 4b adds a + control that every committed revision keeps the previous one's entries, with only `current` becoming + `retired` (rigor pass, CLAIM-9B2-3). - [ ] **Scheduled → Wave 5a: alt-screen word wrapping (#61).** The maintainer approved retaining character-wrap replacement with render-v2/D18 and picker diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index ba9e4df9..2023ad45 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -10380,7 +10380,7 @@ Engineering defaults recorded with the rulings: ### Wave 3 Step 9b.2 implementation checkpoint — 2026-10-25 -Step 9b.2 is implemented on `development` in `84b8e455..4d49176b` and this record, under the +Step 9b.2 is implemented on `development` in the commits from `84b8e455` through `4d49176b` and this record, under the [preflight decisions](#wave-3-step-9b2-preflight-decisions--2026-10-25). Its canonical description is packaging.md's [Step 9b.2 boundary](../../reference/cli/packaging.md#recipient-custody-implementation-boundary--step-9b2). No Step 9b.1 module changed, and nothing outside the new tests calls the new modules. @@ -10395,7 +10395,7 @@ Step 9b.2 is implemented on `development` in `84b8e455..4d49176b` and this recor - **The gate and the sealer** (`release-custody.mjs`). Every scope refusal comes before any GnuPG call, and a module-private registry ties the sealer to the records the gate returned. The receipt is published once through a hard link, and a sibling left after the link is not a refusal. -- **Tests.** Four new files with 34 tests join `test:acceptance-tools`: `release-recipient`, `release-binding`, +- **Tests.** Four new files join `test:acceptance-tools`, with 34 tests before the rigor pass and 39 after it: `release-recipient`, `release-binding`, `release-custody` and `release-scope`. They include a production round trip, in which an ephemeral key under a rotated history decrypts to the container whose manifest the receipt binds, and fault injection after the link and after encryption. @@ -10403,7 +10403,7 @@ Step 9b.2 is implemented on `development` in `84b8e455..4d49176b` and this recor limits (an orphaned ciphertext, no recipient re-check on reuse, Windows not exercised), the Step 9d handoffs and a Wave 4b clause on the recipient's committed form. -**Sweeps.** A generated sweep mutated every one-line refusal, condition clause and comparison of the three modules: +**Sweeps before the rigor pass.** A generated sweep mutated every one-line refusal, condition clause and comparison of the three modules: 131 mutants, 119 killed at first. Five survivors were real gaps and gained cases: - a `pending` status beside the current one; - a one-entry inputs map; @@ -10420,4 +10420,44 @@ The other seven are equivalent: denylist, the root's shape, the current pins, the registry's consumption, the best-effort removal, the `EEXIST` mapping, the canonical bytes, the sealer's fields and the link-aware presence check. -A rigor pass, the full CI and the Ubuntu 2.4.4 evidence follow before the first round. +**Rigor pass at `d94ed705`.** Internal lenses (claims, platform, crypto/custody pre-sitting, tests and integration) +and a critic ran, every finding adversarially verified. The platform lens was clean. Three mediums, corrected: +- **The loader's seams minted a production recipient.** The exported `root` and `fixtures` parameters let any caller + brand any key, a fixture key or the per-run key included, with the denylist switched off; only a comment and an + interim control stood in the way. `loadProductionRecipient()` now takes no argument, and tests load temporary roots + through a separately named `loadRecipientForTests`. Two permanent controls keep every call of the loader + argument-free and its test seam out of product and tooling code (CLAIM-9B2-1, SIT-9B2-4). +- **The import control matched three spellings.** A `../custody.mjs` import from `smoke/`, a workflow's `./tools/…` + path or a query suffix passed unseen. The control now resolves every quoted specifier against its file, or a + workflow's against the repository root, and has its own positive and negative self-test (SIT-9B2-1, T-9B2-R1-1). +- **The primary half of the recipient pair was untested.** A pair matching only by its subkey now refuses in both + scopes (T-9B2-R1-2). + +Lows corrected: +- the gate reads its scope once and encrypts its own copy of the container, and the validator snapshots its + expectation (SIT-9B2-2, T-9B2-R1-3); +- the gate resolves the destination before encrypting (`PLACEMENT`) and checks the tarball against the manifest + (`TARBALL_DRIFT`), so nothing after publication can fail on the destination and the likeliest orphan causes refuse + first (CRIT-9B2-3, SIT-9B2-5); +- an integer-like input name refuses, so the canonical bytes list inputs by path (CLAIM-9B2-5, SIT-9B2-8); +- the fixtures' pins file is parsed strictly (SIT-9B2-3); +- the closure control also reads `export * as`, forbids `cluster` and `module`, and scans the closure's files for + `require`, `createRequire`, `getBuiltinModule` and `process.binding` (SIT-9B2-7, T-9B2-R1-5, INT-9B2-R1-2, + CRIT-9B2-1); +- the guard control uses a complete, valid synthetic candidate (T-9B2-R1-4, INT-9B2-R1-1); +- new tests cover the origin's length bounds, a genuine invalid UTF-8 byte, a reused asset swapped for a link after + its size check, and the reuse refusal after a sibling is left (T-9B2-R1-6, T-9B2-R1-7, T-9B2-R1-8, SIT-9B2-6); +- wording: the contract's diagnostics, canonical-bytes and orphan sentences, the module header, the README and this + checkpoint's commit range (CLAIM-9B2-4, CLAIM-9B2-6, INT-9B2-R1-3 to INT-9B2-R1-6, CRIT-9B2-4). + +Recorded in deferred-tasks.md: the fixture denylist misses an unlisted second subkey, a left sibling blocks reuse +until it is removed, the append-only rule needs a Wave 4b control, and the release job's checkout must include the +fixtures' pins file (CLAIM-9B2-2, CLAIM-9B2-3, SIT-9B2-6, CRIT-9B2-2). + +**Sweeps at the corrected head.** The generated sweep ran 137 mutants and killed 128 at first. One more gap, a +malformed fixture subkey, gained a case. The other eight are equivalent: six history, fixture and read checks whose +removal the loader's wrapper maps to the same code, the inputs comparator, and the receipt bound's `>=` flip. Of 29 +named mutants, 28 are killed. The survivor, the synthetic recipient's snapshot, is equivalent: synthetic scope admits +any recipient, and `encryptOnce` checks the file it reads against the pins it parsed. + +The full CI and the Ubuntu 2.4.4 evidence follow before the first round. diff --git a/tools/cli-package/README.md b/tools/cli-package/README.md index 5a1ac60a..db2695b4 100644 --- a/tools/cli-package/README.md +++ b/tools/cli-package/README.md @@ -77,8 +77,8 @@ is written (`verifySealedDist`). `pnpm check:map-controls` (`map-controls.mjs` o `map-controls-core.mjs`) runs every named negative control on product copies of one real production, prints one JSON summary of codes and fixed lines (no absolute path, frame or map byte) and exits 1 unless every expected control ran once with its expected outcome; the required `map-controls` CI job -runs it uncached and uploads nothing. Step 9b.1's custody library (synthetic recipients only; nothing -calls it yet) builds one owner-only USTAR of the sealed staging (`container.mjs`, over the one strict +runs it uncached and uploads nothing. Step 9b.1's custody library (nothing outside Step 9b.2's release +modules and the tests calls it) builds one owner-only USTAR of the sealed staging (`container.mjs`, over the one strict dialect in `ustar.mjs`) and encrypts it once to one pinned recipient with GnuPG ≥ 2.2.8 in fresh isolated homes (`custody.mjs` over `gnupg.mjs`; POSIX only, since no private root qualifies on Windows): a colon pre-check refuses missing, expired, revoked, unusable or unexpected recipients and the diff --git a/tools/cli-package/release-binding.mjs b/tools/cli-package/release-binding.mjs index aac27d17..0474c84e 100644 --- a/tools/cli-package/release-binding.mjs +++ b/tools/cli-package/release-binding.mjs @@ -2,9 +2,10 @@ * The release binding receipt (Step 9b.2; packaging.md, "Recipient custody implementation boundary — * Step 9b.2"): schema v1, its validator and the verified reuse of a retained candidate. The receipt is * public JSON beside the ciphertext, written once in canonical bytes. It names files by SHA-256 and signs - * nothing: every field is a claim, and its origin, map-check and `ci` fields are checked for shape only - * until Step 9d binds them to the trusted run. Scope always comes from the verifier's expectation, never - * from the receipt. This module's import closure starts no process, so reuse makes no GnuPG call. + * nothing: every field is a claim, and its origin and map-check fields are checked for shape only until + * Step 9d binds them to the trusted run and a successful `ci` run. Scope always comes from the verifier's + * expectation, never from the receipt. This module's import closure starts no process, so reuse makes no + * GnuPG call. */ import { Buffer } from 'node:buffer'; import { lstatSync, readdirSync } from 'node:fs'; @@ -78,20 +79,39 @@ function size(value, minimum, maximum) { if (value < minimum || value > maximum) refuseCustody('BINDING_SIZE'); return value; } -/** A repository-relative POSIX path with no empty, `.` or `..` segment. @param {string} path */ +/** + * A repository-relative POSIX path with no empty, `.` or `..` segment, and not an integer-like name, which + * JavaScript orders ahead of every other key so that the canonical bytes could not list it by path. + * @param {string} path + */ const isRelativePath = (path) => path.length <= 256 && + !/^(?:0|[1-9]\d*)$/u.test(path) && path.split('/').every((segment) => SEGMENT.test(segment) && segment !== '.' && segment !== '..'); -/** @param {unknown} expected */ +/** + * A snapshot of the expectation, each field read once: the admitted recipient pairs, the package version + * and, for production, the source commit. + * @param {unknown} expected + * @returns {{scope: 'production' | 'synthetic', history: readonly {primary: unknown, subkey: unknown}[], + * packageVersion: unknown, source: unknown}} + */ function scopeOf(expected) { if (!isRecord(expected)) return refuseCustody('BINDING_SCOPE'); - if (expected.scope === 'production') { - if (!isProductionRecipient(expected.recipient)) refuseCustody('RECIPIENT_SOURCE'); - } else if (expected.scope !== 'synthetic') { - refuseCustody('BINDING_SCOPE'); + const { scope, recipient, pins, packageVersion, source } = expected; + if (scope === 'production') { + if (!isProductionRecipient(recipient)) refuseCustody('RECIPIENT_SOURCE'); + const { history } = /** @type {ProductionRecipient} */ (recipient); + return Object.freeze({ scope, history, packageVersion, source }); } - return /** @type {Expected} */ (expected); + if (scope !== 'synthetic') refuseCustody('BINDING_SCOPE'); + const pair = isRecord(pins) ? { primary: pins.primary, subkey: pins.subkey } : {}; + return Object.freeze({ + scope, + history: Object.freeze([Object.freeze({ primary: pair.primary, subkey: pair.subkey })]), + packageVersion, + source: undefined, + }); } /** @@ -116,7 +136,11 @@ function asset(value, version, name, minimum, maximum) { * @param {Expected} expected */ export function validateReleaseBinding(value, expected) { - const scope = scopeOf(expected); + return validate(value, scopeOf(expected)); +} + +/** @param {unknown} value @param {ReturnType} scope */ +function validate(value, scope) { if (!isRecord(value) || value.version !== 1 || value.kind !== RELEASE_BINDING_KIND) { refuseCustody('BINDING_VERSION'); } @@ -159,12 +183,8 @@ export function validateReleaseBinding(value, expected) { primary: string(recipientRecord.primary, FINGERPRINT), subkey: string(recipientRecord.subkey, FINGERPRINT), }); - const admitted = - scope.scope === 'production' - ? scope.recipient.history - : [{ primary: scope.pins?.primary, subkey: scope.pins?.subkey }]; if ( - !admitted.some( + !scope.history.some( (entry) => entry.primary === recipient.primary && entry.subkey === recipient.subkey, ) ) { @@ -257,6 +277,11 @@ export const serializeReleaseBinding = (receipt) => * @param {Buffer} bytes @param {Expected} expected */ export function parseReleaseBinding(bytes, expected) { + return parse(bytes, scopeOf(expected)); +} + +/** @param {Buffer} bytes @param {ReturnType} scope */ +function parse(bytes, scope) { if (!Buffer.isBuffer(bytes) || bytes.length > RECEIPT_BYTES) refuseCustody('BINDING_BYTES'); let value; try { @@ -264,7 +289,7 @@ export function parseReleaseBinding(bytes, expected) { } catch { return refuseCustody('BINDING_BYTES'); } - const receipt = validateReleaseBinding(value, expected); + const receipt = validate(value, scope); if (!serializeReleaseBinding(receipt).equals(bytes)) refuseCustody('BINDING_BYTES'); return receipt; } @@ -287,7 +312,8 @@ export function verifyRetainedCandidate({ directory, expected }) { /** @param {string} directory @param {Expected} expected */ function verify(directory, expected) { - const { packageVersion } = scopeOf(expected); + const scope = scopeOf(expected); + const { packageVersion } = scope; if (typeof packageVersion !== 'string' || !PACKAGE_VERSION.test(packageVersion)) { refuseCustody('BINDING_CANDIDATE'); } @@ -303,7 +329,7 @@ function verify(directory, expected) { } catch { refuseCustody('CANDIDATE_ASSET'); } - const receipt = parseReleaseBinding(bytes, expected); + const receipt = parse(bytes, scope); /** @param {{name: string, bytes: number, sha256: string}} entry */ const retained = (entry) => { const path = join(directory, entry.name); diff --git a/tools/cli-package/release-binding.test.mjs b/tools/cli-package/release-binding.test.mjs index ea1a78d6..23be5901 100644 --- a/tools/cli-package/release-binding.test.mjs +++ b/tools/cli-package/release-binding.test.mjs @@ -12,6 +12,7 @@ import { symlinkSync, writeFileSync, } from 'node:fs'; +import { createRequire, syncBuiltinESMExports } from 'node:module'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { after, test } from 'node:test'; @@ -31,7 +32,7 @@ import { validateReleaseBinding, verifyRetainedCandidate, } from './release-binding.mjs'; -import { SYNTHETIC_SOURCE, loadProductionRecipient } from './release-recipient.mjs'; +import { SYNTHETIC_SOURCE, loadRecipientForTests } from './release-recipient.mjs'; const scratch = mkdtempSync(join(tmpdir(), 'relavium-release-binding-')); after(() => rmSync(scratch, { recursive: true, force: true })); @@ -60,7 +61,7 @@ function production(...recipients) { mkdirSync(root); writeFileSync(join(root, 'recipient.asc'), 'key'); writeFileSync(join(root, 'pins.json'), JSON.stringify({ version: 1, recipients })); - return loadProductionRecipient({ root }); + return loadRecipientForTests({ root }); } const ROTATED = production({ ...A, status: 'retired' }, { ...B, status: 'current' }); const synthetic = Object.freeze({ scope: 'synthetic', pins: PINS, packageVersion: VERSION }); @@ -176,12 +177,8 @@ test('every key at every level is exact', () => { 'origin', ]) { for (const key of Object.keys(value[parent])) { - const code = - parent === 'origin' && key === 'kind' - ? 'BINDING_ORIGIN' - : parent === 'origin' && key === 'source' - ? 'BINDING_SHAPE' - : 'BINDING_SHAPE'; + // An origin without its kind is not a GitHub origin. + const code = parent === 'origin' && key === 'kind' ? 'BINDING_ORIGIN' : 'BINDING_SHAPE'; assert.throws( () => validateReleaseBinding( @@ -263,6 +260,14 @@ test('every value has its exact format', () => { (copy) => (copy.origin.runId = '0'), (copy) => (copy.origin.runAttempt = '01'), (copy) => (copy.origin.runAttempt = '1234567'), + (copy) => (copy.origin.repository = `${'a'.repeat(100)}/${'b'.repeat(100)}`), + (copy) => (copy.origin.workflow = 'w'.repeat(201)), + (copy) => (copy.origin.runId = '1'.repeat(21)), + // Integer-like names would sort ahead of every path in the canonical bytes. + ...['0', '9', '10', '4294967295'].map( + (path) => (copy) => + (copy.inputs = { [path]: 'a'.repeat(64), 'package.json': 'b'.repeat(64) }), + ), ]; for (const [index, change] of cases.entries()) { assert.throws( @@ -282,6 +287,15 @@ test('every value has its exact format', () => { }); assert.equal(Object.keys(validateReleaseBinding(bounded, synthetic).inputs).length, 64); const single = changed(value, (copy) => (copy.inputs = { 'package.json': 'a'.repeat(64) })); + // The origin's bounds are inclusive, and a numeric-looking path with a leading zero is a name. + const origins = changed(value, (copy) => { + copy.origin.repository = `${'a'.repeat(99)}/${'b'.repeat(100)}`; + copy.origin.workflow = 'w'.repeat(200); + copy.origin.runId = '1'.repeat(20); + copy.origin.runAttempt = '123456'; + copy.inputs = { '01': 'a'.repeat(64), 'package.json': 'b'.repeat(64) }; + }); + assert.deepEqual(validateReleaseBinding(origins, synthetic).origin, origins.origin); assert.deepEqual(validateReleaseBinding(single, synthetic).inputs, { 'package.json': 'a'.repeat(64), }); @@ -306,6 +320,8 @@ test('the version, candidate, names, sizes, map check and origin refuse with the ['BINDING_ORIGIN', (copy) => (copy.origin.source = 'f'.repeat(40))], ['BINDING_RECIPIENT', (copy) => (copy.recipient = { ...C })], ['BINDING_RECIPIENT', (copy) => (copy.recipient.subkey = C.subkey)], + // Both halves of the pair must match: the expected subkey under another primary refuses. + ['BINDING_RECIPIENT', (copy) => (copy.recipient.primary = C.primary)], ['BINDING_SCOPE', (copy) => (copy.source = { ...CLEAN })], ['BINDING_SCOPE', (copy) => (copy.source.tree = 'e'.repeat(40))], ]) { @@ -337,6 +353,7 @@ test('production scope admits only a clean, non-synthetic source of the expected ['BINDING_DIRTY', (copy) => (copy.source.dirty = true)], ['BINDING_RECIPIENT', (copy) => (copy.recipient = { ...C })], ['BINDING_RECIPIENT', (copy) => (copy.recipient = { primary: A.primary, subkey: C.subkey })], + ['BINDING_RECIPIENT', (copy) => (copy.recipient = { primary: C.primary, subkey: B.subkey })], ]) { assert.throws( () => @@ -420,8 +437,12 @@ test('only the exact canonical bytes parse', () => { ['trailing byte', Buffer.concat([bytes, Buffer.from(' ')])], ['CRLF', Buffer.from(text.replaceAll('\n', '\r\n'))], [ - 'invalid UTF-8', - Buffer.concat([bytes.subarray(0, -2), Buffer.from([0xff]), Buffer.from('\n')]), + 'invalid UTF-8 inside a string', + (() => { + const bad = Buffer.from(bytes); + bad[bytes.indexOf('"workflow": "release"') + '"workflow": "r'.length] = 0xff; + return bad; + })(), ], ['not JSON', Buffer.from('receipt')], ['too large', Buffer.concat([bytes, Buffer.alloc(64 * 1024 + 1 - bytes.length, 0x20)])], @@ -673,6 +694,60 @@ test('a retained candidate whose assets or entries differ from its receipt refus ); }); +test('each expectation field is read once, so a changing expectation cannot change scope midway', () => { + let reads = 0; + const flipping = { + get scope() { + reads += 1; + return reads === 1 ? 'synthetic' : 'production'; + }, + recipient: ROTATED, + pins: PINS, + packageVersion: VERSION, + source: CLEAN.commit, + }; + // Read as synthetic once: a clean source is never a synthetic candidate. + assert.throws( + () => validateReleaseBinding(receipt({ source: CLEAN, recipient: PINS }), flipping), + refuses('BINDING_SCOPE'), + ); + assert.equal(reads, 1); +}); + +test( + 'a retained asset swapped for a link after its size check refuses as not an ordinary file', + { skip: !posix }, + () => { + const directory = candidate(); + const path = join(directory, ciphertextFileName(VERSION)); + const other = fresh('other-ciphertext'); + writeFileSync(other, Buffer.alloc(CIPHERTEXT.length, 0x78)); + const fs = createRequire(import.meta.url)('node:fs'); + const lstat = fs.lstatSync; + let swapped = false; + fs.lstatSync = (target, ...rest) => { + const info = lstat(target, ...rest); + if (target === path && !swapped) { + swapped = true; + rmSync(path); + symlinkSync(other, path); + } + return info; + }; + syncBuiltinESMExports(); + try { + assert.throws( + () => verifyRetainedCandidate({ directory, expected: synthetic }), + refuses('CANDIDATE_ASSET'), + ); + } finally { + fs.lstatSync = lstat; + syncBuiltinESMExports(); + } + assert.equal(swapped, true); + }, +); + test('reuse needs a valid expectation, and a host failure refuses with a fixed code and no path', () => { const directory = candidate(); for (const [code, expected] of [ diff --git a/tools/cli-package/release-custody.mjs b/tools/cli-package/release-custody.mjs index d06ddee8..bb628d67 100644 --- a/tools/cli-package/release-custody.mjs +++ b/tools/cli-package/release-custody.mjs @@ -6,6 +6,7 @@ * encrypt a product staging. The sealer accepts only a record this gate returned, and writes the receipt * once, after re-reading the ciphertext it names. This is the only release module that reaches GnuPG. */ +import { Buffer } from 'node:buffer'; import { randomUUID } from 'node:crypto'; import { closeSync, @@ -43,8 +44,8 @@ import { isProductionRecipient, isSyntheticSource } from './release-recipient.mj /** * The records the gate returned, with what the sealer needs to bind them. - * @type {WeakMap} + * @type {WeakMap} */ const records = new WeakMap(); @@ -86,52 +87,68 @@ export async function encryptCandidate({ if (error instanceof CustodyError) throw error; refuseCustody('UNEXPECTED'); } + // The admitted copy is the one encrypted: the caller's buffer may change, it may not. const record = await encryptOnce({ - container, + container: admitted.container, recipient: admitted.recipient, - destination, + destination: admitted.destination, privateRoot, now, run, gpg, }); - try { - records.set(record, { - scope, - manifest: admitted.manifest, - destination: realpathSync(destination), - }); - } catch { - refuseCustody('UNEXPECTED'); - } + records.set(record, { + kind: admitted.kind, + recipient: admitted.production, + manifest: admitted.manifest, + destination: admitted.destination, + }); return record; } -/** @param {unknown} scope @param {Buffer} container @param {string} destination */ +/** + * Admit one encryption, reading the caller's scope and container once: the scope's rules, an unsealed + * destination and the candidate's tarball, all before any GnuPG call. + * @param {unknown} scope @param {unknown} container @param {string} destination + */ function admit(scope, container, destination) { - if ( - !isRecord(scope) || - (scope.kind !== 'production' && scope.kind !== 'synthetic') || - !isRecord(scope.recipient) - ) { - refuseCustody('SCOPE'); - } - if (scope.kind === 'production' && !isProductionRecipient(scope.recipient)) { - refuseCustody('RECIPIENT_SOURCE'); - } - const { manifest } = readPrivateContainer(container); + const { kind, recipient: given } = isRecord(scope) ? scope : {}; + if ((kind !== 'production' && kind !== 'synthetic') || !isRecord(given)) refuseCustody('SCOPE'); + if (kind === 'production' && !isProductionRecipient(given)) refuseCustody('RECIPIENT_SOURCE'); + const bytes = Buffer.isBuffer(container) ? Buffer.from(container) : container; + const { manifest } = readPrivateContainer(/** @type {Buffer} */ (bytes)); let recipient; - if (scope.kind === 'production') { + let production; + if (kind === 'production') { if (manifest.source.dirty) refuseCustody('DIRTY_SOURCE'); if (isSyntheticSource(manifest.source)) refuseCustody('SYNTHETIC_SOURCE'); - const production = /** @type {ProductionRecipient} */ (scope.recipient); + production = /** @type {ProductionRecipient} */ (given); recipient = { file: production.file, pins: production.current }; } else { if (!isSyntheticSource(manifest.source)) refuseCustody('FIXTURE_SCOPE'); - recipient = /** @type {{file: string, pins: unknown}} */ (scope.recipient); + recipient = { file: given.file, pins: given.pins }; } if (present(join(destination, RELEASE_BINDING_FILE))) refuseCustody('ALREADY_SEALED'); - return { manifest, recipient }; + // Resolved now, so nothing after the ciphertext is published can fail on the destination. + let resolved; + try { + resolved = realpathSync(destination); + } catch { + refuseCustody('PLACEMENT'); + } + // The receipt binds the tarball the manifest names; a missing or different one would leave an + // unsealable ciphertext, so it refuses first. + let tarball; + try { + tarball = readOrdinaryFile( + join(resolved, tarballName(manifest.packageVersion)), + ARCHIVE_LIMITS.compressedBytes, + ); + } catch { + refuseCustody('TARBALL_DRIFT'); + } + if (sha256(tarball) !== manifest.tarballSha256) refuseCustody('TARBALL_DRIFT'); + return { kind, recipient, production, manifest, container: bytes, destination: resolved }; } /** Write a new public file durably; an existing file refuses. @param {string} path @param {Buffer} bytes */ @@ -198,7 +215,7 @@ export function sealReleaseBinding(record, { origin, write = writePublicFile } = function seal(record, origin, write) { const entry = records.get(record); if (entry === undefined) refuseCustody('RECORD_SOURCE'); - const { scope, manifest, destination } = entry; + const { kind, recipient, manifest, destination } = entry; // An existing receipt refuses before anything is read; one is never written over. if (present(join(destination, RELEASE_BINDING_FILE))) refuseCustody('ALREADY_SEALED'); const tarball = tarballName(manifest.packageVersion); @@ -225,10 +242,10 @@ function seal(record, origin, write) { if (tarballSha256 !== manifest.tarballSha256) refuseCustody('TARBALL_DRIFT'); const expected = - scope.kind === 'production' + kind === 'production' ? { scope: /** @type {const} */ ('production'), - recipient: scope.recipient, + recipient, packageVersion: manifest.packageVersion, source: manifest.source.commit, } diff --git a/tools/cli-package/release-custody.test.mjs b/tools/cli-package/release-custody.test.mjs index 6fcde1be..1434f179 100644 --- a/tools/cli-package/release-custody.test.mjs +++ b/tools/cli-package/release-custody.test.mjs @@ -39,7 +39,7 @@ import { places, recipientRoot, } from './release-fixture.mjs'; -import { SYNTHETIC_SOURCE, loadProductionRecipient } from './release-recipient.mjs'; +import { SYNTHETIC_SOURCE, loadRecipientForTests } from './release-recipient.mjs'; const refuses = (code) => (error) => error instanceof CustodyError && @@ -53,10 +53,11 @@ before(async () => { }); /** A run port that records every GnuPG call; everything else is the real GnuPG. */ -function recording() { +function recording(before = () => {}) { const calls = []; /** @type {typeof runCommand} */ const run = async (command, args, options) => { + before(args); calls.push( args.find( (arg) => @@ -99,7 +100,7 @@ test('the gate refuses a wrong scope, recipient or candidate before any GnuPG ca const ephemeral = ephemeralKey(); const production = { kind: 'production', - recipient: loadProductionRecipient({ + recipient: loadRecipientForTests({ root: recipientRoot(ephemeral.file, [{ ...ephemeral.pins, status: 'current' }]), }), }; @@ -127,6 +128,58 @@ test('the gate refuses a wrong scope, recipient or candidate before any GnuPG ca assert.deepEqual(calls, [], code); assert.deepEqual(readdirSync(destination), [tarballName(VERSION)], code); } + // A destination that cannot be resolved, or whose tarball is missing or another, refuses before any + // call: a ciphertext encrypted into it could never be sealed. + for (const [code, change] of [ + ['PLACEMENT', (destination) => rmSync(destination, { recursive: true })], + ['TARBALL_DRIFT', (destination) => rmSync(join(destination, tarballName(VERSION)))], + [ + 'TARBALL_DRIFT', + (destination) => + writeFileSync(join(destination, tarballName(VERSION)), Buffer.from('another tarball')), + ], + ]) { + const { root, destination } = places(); + change(destination); + const { run, calls } = recording(); + await assert.rejects( + encryptCandidate({ + scope: synthetic, + container: built.synthetic, + destination, + privateRoot: root, + run, + }), + refuses(code), + code, + ); + assert.deepEqual(calls, [], code); + } + // The scope is read once: a kind that changes after its first read cannot skip the brand check. + { + let reads = 0; + const flipping = { + get kind() { + reads += 1; + return reads === 1 ? 'production' : 'synthetic'; + }, + recipient: { file: ephemeral.file, current: ephemeral.pins, history: [] }, + }; + const { root, destination } = places(); + const { run, calls } = recording(); + await assert.rejects( + encryptCandidate({ + scope: flipping, + container: built.clean, + destination, + privateRoot: root, + run, + }), + refuses('RECIPIENT_SOURCE'), + ); + assert.deepEqual(calls, []); + assert.equal(reads, 1); + } // A destination that already holds a receipt is sealed: nothing is encrypted into it. for (const kind of ['file', 'dangling link']) { const { root, destination } = places(); @@ -396,23 +449,33 @@ test( syncBuiltinESMExports(); chmodSync(destination, 0o700); } + // Published, but not reusable until the sibling is removed: reuse admits exactly three entries. + assert.throws( + () => + verifyRetainedCandidate({ + directory: destination, + expected: { scope: 'synthetic', pins: PINS.good, packageVersion: VERSION }, + }), + refuses('CANDIDATE_ENTRIES'), + ); }, ); -test('a host failure after encryption refuses with a fixed code and no path', async () => { +test('a destination that stops resolving refuses before any GnuPG call, with a fixed code and no path', async () => { const { root, destination } = places(); const fs = createRequire(import.meta.url)('node:fs'); const realpath = fs.realpathSync; let calls = 0; - // encryptOnce resolves the destination once; the gate's second resolution fails. + // The gate resolves the destination once, before encryption: nothing after publication can fail on it. fs.realpathSync = Object.assign( (path, ...rest) => { - if (path === destination && (calls += 1) === 2) throw new Error(`vanished: ${destination}`); + if (path === destination && (calls += 1) === 1) throw new Error(`vanished: ${destination}`); return realpath(path, ...rest); }, { native: realpath.native }, ); syncBuiltinESMExports(); + const recorded = recording(); try { await assert.rejects( encryptCandidate({ @@ -420,21 +483,47 @@ test('a host failure after encryption refuses with a fixed code and no path', as container: built.synthetic, destination, privateRoot: root, + run: recorded.run, }), - (error) => refuses('UNEXPECTED')(error) && !error.message.includes(destination), + (error) => refuses('PLACEMENT')(error) && !error.message.includes(destination), ); } finally { fs.realpathSync = realpath; syncBuiltinESMExports(); } - assert.equal(calls, 2); + assert.equal(calls, 1); + assert.deepEqual(recorded.calls, []); +}); + +test("the caller's container changing during encryption changes nothing encrypted or sealed", async () => { + const ephemeral = ephemeralKey(); + const scope = { kind: 'synthetic', recipient: { file: ephemeral.file, pins: ephemeral.pins } }; + const mine = Buffer.from(built.synthetic); + // Overwritten in place while GnuPG runs, before the gate's copy is written for encryption. + const { run } = recording((args) => { + if (args.includes('--show-keys')) mine.fill(0x20); + }); + const { root, destination } = places(); + const record = await encryptCandidate({ + scope, + container: mine, + destination, + privateRoot: root, + run, + }); + assert.equal(mine[0], 0x20); + const out = fresh('release-copy'); + mkdirSync(out); + ephemeral.decrypt(join(destination, record.name), join(out, 'container.tar')); + assert.deepEqual(readFileSync(join(out, 'container.tar')), built.synthetic); + sealReleaseBinding(record, { origin: origin(SYNTHETIC_SOURCE.commit) }); }); test('a production candidate round-trips: the receipt binds what the ciphertext decrypts to', async () => { const ephemeral = ephemeralKey(); // A rotated history: an earlier recipient retired, the ephemeral key current. const earlier = { primary: '1'.repeat(40), subkey: '2'.repeat(40) }; - const recipient = loadProductionRecipient({ + const recipient = loadRecipientForTests({ root: recipientRoot(ephemeral.file, [ { ...earlier, status: 'retired' }, { ...ephemeral.pins, status: 'current' }, @@ -475,7 +564,7 @@ test('a production candidate round-trips: the receipt binds what the ciphertext }), refuses('BINDING_SCOPE'), ); - const rotatedAway = loadProductionRecipient({ + const rotatedAway = loadRecipientForTests({ root: recipientRoot(ephemeral.file, [{ ...earlier, status: 'current' }]), }); assert.throws( diff --git a/tools/cli-package/release-recipient.mjs b/tools/cli-package/release-recipient.mjs index 8107f815..45f79563 100644 --- a/tools/cli-package/release-recipient.mjs +++ b/tools/cli-package/release-recipient.mjs @@ -89,7 +89,11 @@ function parseHistory(bytes) { return Object.freeze(history); } -/** Every fingerprint the committed fixtures name. @param {string} file */ +/** + * Every fingerprint the committed fixtures' pins file lists. A file that is not exactly a non-empty map of + * named primary and subkey pins refuses rather than yield a weaker denylist. + * @param {string} file + */ function fixtureFingerprints(file) { let pins; try { @@ -97,21 +101,43 @@ function fixtureFingerprints(file) { } catch { return refuseCustody('PRODUCTION_RECIPIENT_INVALID'); } - if (!isRecord(pins)) refuseCustody('PRODUCTION_RECIPIENT_INVALID'); - return new Set( - Object.values(pins).flatMap((entry) => (isRecord(entry) ? [entry.primary, entry.subkey] : [])), - ); + const entries = isRecord(pins) ? Object.values(pins) : []; + if ( + entries.length === 0 || + !entries.every( + (entry) => + isRecord(entry) && + isDeepStrictEqual(Object.keys(entry).sort(), ['primary', 'subkey']) && + isFingerprint(entry.primary) && + isFingerprint(entry.subkey), + ) + ) { + refuseCustody('PRODUCTION_RECIPIENT_INVALID'); + } + return new Set(entries.flatMap((entry) => [entry.primary, entry.subkey])); +} + +/** + * Load the production recipient from the committed directory. It takes no argument: there is no other + * place a production recipient can come from. + * @returns {ProductionRecipient} + */ +export function loadProductionRecipient() { + return guarded(PRODUCTION_RECIPIENT_ROOT, FIXTURE_PINS_FILE); } /** - * Load the production recipient. `root` and `fixtures` are test seams only; no product path passes them. - * @param {{root?: string, fixtures?: string}} [options] + * The tests' seam: load a recipient from a temporary directory. No product or tooling code may call it; + * release-scope.test.mjs pins that permanently. + * @param {{root: string, fixtures?: string}} options * @returns {ProductionRecipient} */ -export function loadProductionRecipient({ - root = PRODUCTION_RECIPIENT_ROOT, - fixtures = FIXTURE_PINS_FILE, -} = {}) { +export function loadRecipientForTests({ root, fixtures = FIXTURE_PINS_FILE } = {}) { + return guarded(root, fixtures); +} + +/** @param {string} root @param {string} fixtures @returns {ProductionRecipient} */ +function guarded(root, fixtures) { try { return load(root, fixtures); } catch (error) { diff --git a/tools/cli-package/release-recipient.test.mjs b/tools/cli-package/release-recipient.test.mjs index fd5b727f..36543a3f 100644 --- a/tools/cli-package/release-recipient.test.mjs +++ b/tools/cli-package/release-recipient.test.mjs @@ -15,6 +15,7 @@ import { isProductionRecipient, isSyntheticSource, loadProductionRecipient, + loadRecipientForTests, } from './release-recipient.mjs'; const scratch = mkdtempSync(join(tmpdir(), 'relavium-release-recipient-')); @@ -54,15 +55,20 @@ test('the committed recipient directory is absent in Wave 3, so every production fileURLToPath(new URL('.', import.meta.url)).replace(/[\\/]$/u, ''), ); assert.throws(() => loadProductionRecipient(), refuses('PRODUCTION_RECIPIENT_ABSENT')); + // It takes no argument: a root passed to it is ignored, never an override. assert.throws( - () => loadProductionRecipient({ root: join(fresh('missing'), 'release-recipient') }), + () => loadProductionRecipient({ root: root() }), + refuses('PRODUCTION_RECIPIENT_ABSENT'), + ); + assert.throws( + () => loadRecipientForTests({ root: join(fresh('missing'), 'release-recipient') }), refuses('PRODUCTION_RECIPIENT_ABSENT'), ); }); test('a loaded recipient is frozen, branded, and its current pins come from an append-only history', () => { const path = root({ pins: history({ ...A, status: 'retired' }, { ...B, status: 'current' }) }); - const recipient = loadProductionRecipient({ root: path }); + const recipient = loadRecipientForTests({ root: path }); assert.equal(recipient.file, join(path, 'recipient.asc')); assert.deepEqual(recipient.current, B); assert.deepEqual(recipient.history, [ @@ -84,7 +90,7 @@ test('a loaded recipient is frozen, branded, and its current pins come from an a assert.equal(isProductionRecipient(value), false); } // Each load brands its own value. - const again = loadProductionRecipient({ root: path }); + const again = loadRecipientForTests({ root: path }); assert.notEqual(again, recipient); assert.equal(isProductionRecipient(again), true); }); @@ -141,7 +147,7 @@ test('a recipient directory that is not exactly the two committed files refuses' } for (const [index, make] of cases.entries()) { assert.throws( - () => loadProductionRecipient({ root: make() }), + () => loadRecipientForTests({ root: make() }), refuses('PRODUCTION_RECIPIENT_INVALID'), String(index), ); @@ -178,7 +184,7 @@ test('the pins history is exactly one current recipient among at most 32 distinc ), ]) { assert.throws( - () => loadProductionRecipient({ root: root({ pins }) }), + () => loadRecipientForTests({ root: root({ pins }) }), refuses('PRODUCTION_RECIPIENT_INVALID'), typeof pins === 'string' ? pins : JSON.stringify(pins).slice(0, 120), ); @@ -192,7 +198,7 @@ test('the pins history is exactly one current recipient among at most 32 distinc status: 'retired', })), ); - assert.equal(loadProductionRecipient({ root: root({ pins: full }) }).history.length, 32); + assert.equal(loadRecipientForTests({ root: root({ pins: full }) }).history.length, 32); }); test('a history naming a committed fixture key refuses, whichever fingerprint it is', () => { @@ -203,7 +209,7 @@ test('a history naming a committed fixture key refuses, whichever fingerprint it ]) { assert.throws( () => - loadProductionRecipient({ + loadRecipientForTests({ root: root({ pins: history({ ...entry, status: 'current' }) }), }), refuses('FIXTURE_RECIPIENT'), @@ -212,7 +218,7 @@ test('a history naming a committed fixture key refuses, whichever fingerprint it // A retired fixture entry refuses as well. assert.throws( () => - loadProductionRecipient({ + loadRecipientForTests({ root: root({ pins: history({ ...A, status: 'current' }, { ...PINS.other, status: 'retired' }), }), @@ -224,11 +230,24 @@ test('a history naming a committed fixture key refuses, whichever fingerprint it FIXTURE_PINS_FILE, fileURLToPath(new URL('./custody-fixtures/pins.json', import.meta.url)), ); - const malformed = fresh('fixtures.json'); - writeFileSync(malformed, '[]'); - for (const fixtures of [join(fresh('none'), 'pins.json'), malformed]) { + const fixtureFiles = [join(fresh('none'), 'pins.json')]; + for (const content of [ + '[]', + '{}', + 'not json', + JSON.stringify({ good: { primary: A.primary } }), + JSON.stringify({ good: { ...A, extra: 'x' } }), + JSON.stringify({ good: { primary: 'a'.repeat(40), subkey: A.subkey } }), + JSON.stringify({ good: { primary: A.primary, subkey: 'a'.repeat(40) } }), + JSON.stringify({ good: A, bad: null }), + ]) { + const file = fresh('fixtures.json'); + writeFileSync(file, content); + fixtureFiles.push(file); + } + for (const fixtures of fixtureFiles) { assert.throws( - () => loadProductionRecipient({ root: root(), fixtures }), + () => loadRecipientForTests({ root: root(), fixtures }), refuses('PRODUCTION_RECIPIENT_INVALID'), ); } @@ -242,7 +261,7 @@ test( chmodSync(path, 0o000); try { assert.throws( - () => loadProductionRecipient({ root: path }), + () => loadRecipientForTests({ root: path }), refuses('PRODUCTION_RECIPIENT_INVALID'), ); } finally { diff --git a/tools/cli-package/release-scope.test.mjs b/tools/cli-package/release-scope.test.mjs index 22d90236..7fe73a18 100644 --- a/tools/cli-package/release-scope.test.mjs +++ b/tools/cli-package/release-scope.test.mjs @@ -1,4 +1,5 @@ import assert from 'node:assert/strict'; +import { Buffer } from 'node:buffer'; import { spawnSync } from 'node:child_process'; import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { dirname, join, posix, relative } from 'node:path'; @@ -7,11 +8,15 @@ import { fileURLToPath } from 'node:url'; import { COMMIT, TREE, fresh } from './candidate-fixture.mjs'; import { CONTAINER_LIMITS } from './container.mjs'; import { ciphertextName } from './custody.mjs'; +import { sha256 } from './files.mjs'; import { CIPHERTEXT_BYTES, RELEASE_BINDING_FILE, ciphertextFileName, + serializeReleaseBinding, tarballName, + validateReleaseBinding, + verifyRetainedCandidate, } from './release-binding.mjs'; import { ReleaseGuardError, requireReleaseCandidate } from './release-guard-core.mjs'; import { PRODUCTION_RECIPIENT_ROOT, SYNTHETIC_SOURCE } from './release-recipient.mjs'; @@ -75,11 +80,60 @@ test('no product or tooling code reaches a test fixture, and only the loader rea assert.deepEqual(naming(/custody-fixtures/u), ['tools/cli-package/release-recipient.mjs']); }); -test('only the release gate imports encryptOnce', () => { - assert.deepEqual( - naming(/['"](?:\.\/|\.\.\/cli-package\/|tools\/cli-package\/)custody\.mjs['"]/u), - ['tools/cli-package/release-custody.mjs'], - ); +/** + * Whether `text`, a file at repository-relative `path`, names repository-relative `target` by any quoted + * specifier that resolves to it: relative to the file, or to the repository root for a workflow (whose + * steps run there) and for a `tools/` specifier; a query suffix is ignored. + * @param {string} path @param {string} text @param {string} target + */ +function resolvesTo(path, text, target) { + const name = posix.basename(target).replaceAll('.', '\\.'); + const literal = new RegExp(`['"\`]([^'"\`\\s]*(?:^|/)${name})(?:\\?[^'"\`\\s]*)?['"\`]`, 'gu'); + const root = /\.ya?ml$/u.test(path) ? '.' : posix.dirname(path); + for (const match of text.matchAll(literal)) { + const specifier = match[1]; + const base = specifier.startsWith('tools/') ? '.' : root; + const resolved = posix.normalize(posix.join(base, specifier)); + if (resolved === target) return true; + } + return false; +} + +test('the import match resolves every spelling, and only the real target', () => { + const target = 'tools/cli-package/custody.mjs'; + for (const [path, text] of [ + ['tools/cli-package/smoke/x.mjs', "import { encryptOnce } from '../custody.mjs';"], + ['tools/x.mjs', "import { encryptOnce } from './cli-package/custody.mjs';"], + ['tools/ci/x/y.mjs', 'const m = await import("../../cli-package/custody.mjs?v=1");'], + ['.github/workflows/x.yml', 'run: node -e "import(\'./tools/cli-package/custody.mjs\')"'], + ['apps/cli/x.ts', "export * as custody from '../../tools/cli-package/custody.mjs';"], + ['tools/cli-package/x.mjs', 'require(`./custody.mjs`)'], + ]) { + assert.equal(resolvesTo(path, text, target), true, `${path}: ${text}`); + } + for (const [path, text] of [ + ['tools/cli-package/x.mjs', "import { refuseCustody } from './custody-error.mjs';"], + ['tools/cli-package/x.mjs', "import { encryptCandidate } from './release-custody.mjs';"], + ['tools/x.mjs', "import { x } from './custody.mjs';"], + ['tools/cli-package/x.mjs', '// encryptOnce lives in custody.mjs'], + ]) { + assert.equal(resolvesTo(path, text, target), false, `${path}: ${text}`); + } +}); + +test('only the release gate imports encryptOnce, by any spelling', () => { + const target = 'tools/cli-package/custody.mjs'; + assert.deepEqual(code.filter((path) => resolvesTo(path, read(path), target)).sort(), [ + 'tools/cli-package/release-custody.mjs', + ]); +}); + +test('the loader takes no argument, and no product or tooling code reaches its test seam', () => { + // Permanent: these controls stay when Step 9d updates the interim one below. + assert.deepEqual(naming(/\bloadRecipientForTests\b/u), [ + 'tools/cli-package/release-recipient.mjs', + ]); + assert.deepEqual(naming(/\bloadProductionRecipient\s*\(\s*[^)\s]/u), []); }); test('no product path or workflow names a release module yet', () => { @@ -99,7 +153,7 @@ function closure(entry) { seen.add(path); const source = readFileSync(path, 'utf8'); for (const match of source.matchAll( - /^\s*import\s+(?:[^'";]*?\s+from\s+)?['"]([^'"]+)['"]|^\s*export\s+(?:\*|\{[^}]*\})\s+from\s+['"]([^'"]+)['"]|\bimport\(\s*['"]([^'"]+)['"]\s*\)/gmu, + /^\s*import\s+(?:[^'";]*?\s+from\s+)?['"]([^'"]+)['"]|^\s*export\s+(?:\*(?:\s+as\s+\w+)?|\{[^}]*\})\s+from\s+['"]([^'"]+)['"]|\bimport\(\s*['"]([^'"]+)['"]\s*\)/gmu, )) { const specifier = match[1] ?? match[2] ?? match[3]; specifiers.add(specifier); @@ -123,8 +177,18 @@ test("the binding receipt's closure starts no process, so reuse makes no GnuPG c 'release-recipient.mjs', 'ustar.mjs', ]); - for (const forbidden of ['node:child_process', 'child_process', 'node:worker_threads']) { + for (const forbidden of ['child_process', 'worker_threads', 'cluster', 'module']) { assert.equal(specifiers.includes(forbidden), false, forbidden); + assert.equal(specifiers.includes(`node:${forbidden}`), false, forbidden); + } + // Nor can a closure file reach a process without importing one. + for (const file of files) { + const source = readFileSync(join(here, file), 'utf8'); + assert.doesNotMatch( + source, + /\b(?:getBuiltinModule|createRequire|process\.binding|process\.dlopen)\b|\brequire\s*\(/u, + file, + ); } // The scan reads the imports it claims to: the gate's closure reaches GnuPG. const gate = closure('release-custody.mjs'); @@ -146,12 +210,54 @@ test('no production recipient is committed in Wave 3', () => { assert.equal(listed.stdout, ''); }); -test('the interim release guard still refuses a complete synthetic candidate under the receipt name', () => { +test('the interim release guard still refuses a complete, valid synthetic candidate', () => { const directory = fresh('guarded-candidate'); mkdirSync(directory); - writeFileSync(join(directory, RELEASE_BINDING_FILE), '{}'); - writeFileSync(join(directory, tarballName('0.1.1')), 'tarball'); - writeFileSync(join(directory, ciphertextFileName('0.1.1')), 'ciphertext'); + const tarball = Buffer.from('a synthetic tarball, never a package\n'); + const ciphertext = Buffer.from('synthetic ciphertext bytes'); + const pins = { primary: '4'.repeat(40), subkey: '5'.repeat(40) }; + const expected = { scope: 'synthetic', pins, packageVersion: '0.1.1' }; + const receipt = validateReleaseBinding( + { + version: 1, + kind: 'release_binding', + packageVersion: '0.1.1', + source: { ...SYNTHETIC_SOURCE, dirty: true }, + tarball: { name: tarballName('0.1.1'), bytes: tarball.length, sha256: sha256(tarball) }, + ciphertext: { + name: ciphertextFileName('0.1.1'), + bytes: ciphertext.length, + sha256: sha256(ciphertext), + }, + recipient: pins, + toolchain: { + node: 'v22.23.1', + pnpm: '9.12.3', + typescript: '5.6.3', + tsup: '8.3.5', + esbuild: '0.24.0', + gnupg: '2.5.21', + }, + target: 'node22', + inputs: { 'package.json': 'a'.repeat(64) }, + mapCheck: { status: 'complete' }, + origin: { + kind: 'github', + repository: 'example/relavium', + workflow: 'release', + runId: '1', + runAttempt: '1', + event: 'push', + source: SYNTHETIC_SOURCE.commit, + }, + }, + expected, + ); + writeFileSync(join(directory, RELEASE_BINDING_FILE), serializeReleaseBinding(receipt)); + writeFileSync(join(directory, tarballName('0.1.1')), tarball); + writeFileSync(join(directory, ciphertextFileName('0.1.1')), ciphertext); + // The candidate is complete and reusable as it stands, and the guard still refuses it. + verifyRetainedCandidate({ directory, expected }); assert.throws( () => requireReleaseCandidate(directory), (error) => error instanceof ReleaseGuardError && error.code === 'RELEASE_CUSTODY_UNAVAILABLE', From 5cda645948e1b246a9428cb4ac04be62d24941aa Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 26 Oct 2026 01:15:28 +0300 Subject: [PATCH 84/97] fix(repo): address the first Step 9b.2 review round Round 1 at 91d08a65 reported three mediums: - the rigor pass's destination fix handed encryptOnce the resolved path, which dropped 9b.1's refusal of a linked destination; the gate now refuses a link itself and passes encryptOnce the caller's destination, so its refusals apply unchanged; - the gate checked the tarball but not the destination's other entries, so a stray entry led to an unsealable ciphertext; it now requires exactly the tarball (ALREADY_ENCRYPTED, CANDIDATE_ENTRIES) before any GnuPG call; - a bare custody.mjs specifier escaped the import control; the control now reads every quoted literal, nested ones included, decodes percent-escapes, ignores queries and fragments, resolves bare names and scans package scripts. Lows corrected: the validator reads each field once and the public entry points refuse malformed calls and throwing accessors with fixed codes; a vanished reused asset refuses CANDIDATE_ASSET; pins.json must be its canonical serialization; the closure control is an allowlist and refuses non-literal dynamic imports; the fixture control covers every *-fixture.mjs; the overhead parity reads custody.mjs; new tests and wording. Two lows are recorded in deferred-tasks.md. Refs: #46, ADR-0111 Co-Authored-By: Claude Opus 5.5 --- docs/reference/cli/packaging.md | 26 +++-- docs/roadmap/deferred-tasks.md | 14 ++- .../phase-2.5.5-hardening-and-remediation.md | 60 +++++++++- tools/cli-package/release-binding.mjs | 57 +++++---- tools/cli-package/release-binding.test.mjs | 110 +++++++++++++++++- tools/cli-package/release-custody.mjs | 50 ++++---- tools/cli-package/release-custody.test.mjs | 81 ++++++++++++- tools/cli-package/release-fixture.mjs | 6 +- tools/cli-package/release-recipient.mjs | 35 +++--- tools/cli-package/release-recipient.test.mjs | 16 ++- tools/cli-package/release-scope.test.mjs | 67 ++++++++--- 11 files changed, 421 insertions(+), 101 deletions(-) diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 281db75a..11cac1bf 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -640,7 +640,10 @@ refuses every artifact. Its review and acceptance are recorded in the does not exist and the loader refuses `PRODUCTION_RECIPIENT_ABSENT`; a linked or malformed directory, and any host failure while loading, refuses `PRODUCTION_RECIPIENT_INVALID`. A history naming a fingerprint that the committed fixtures' `pins.json` lists refuses `FIXTURE_RECIPIENT`, and - a malformed fixtures file refuses rather than weaken that denylist. The loaded value is frozen and + a malformed fixtures file refuses rather than weaken that denylist. `pins.json` must be exactly its + canonical serialization (`JSON.stringify(value, null, 2)` plus a newline, fields in the order above), + so a duplicated key, which `JSON.parse` would resolve silently to its last value, refuses and the file a + reviewer reads is the history loaded. The loaded value is frozen and branded: only that value counts as a production recipient, never a copy. Tests load temporary roots through the separately named `loadRecipientForTests`, which `release-scope.test.mjs` permanently keeps out of every product and tooling file. Wave 4b commits the files. @@ -649,12 +652,14 @@ refuses every artifact. Its review and acceptance are recorded in the loaded recipient and a clean, non-synthetic candidate, and encrypts to the recipient's current pins (`RECIPIENT_SOURCE`, `DIRTY_SOURCE`, `SYNTHETIC_SOURCE`). It admits synthetic scope only with a synthetic candidate (`FIXTURE_SCOPE`), so neither a fixture key nor the per-run round-trip key can - encrypt a product staging. An unknown scope refuses `SCOPE`, a destination already holding a - receipt refuses `ALREADY_SEALED`, a destination that does not resolve refuses `PLACEMENT`, and a - destination without the tarball the manifest names refuses `TARBALL_DRIFT`, so a ciphertext is - never encrypted where it could not be sealed. Every one of these refuses before any GnuPG call. - The gate reads the caller's scope once and encrypts its own copy of the container, so neither can - change midway; `encryptOnce` then runs unchanged on that copy. + encrypt a product staging. An unknown scope refuses `SCOPE`. The destination must be a plain, + resolvable directory (`PLACEMENT`; a link refuses, as `encryptOnce` requires) holding exactly the + tarball the manifest names: a receipt there refuses `ALREADY_SEALED`, the ciphertext + `ALREADY_ENCRYPTED`, any other entry, or a tarball named otherwise, `CANDIDATE_ENTRIES`, and a different + tarball `TARBALL_DRIFT`. A ciphertext is therefore never encrypted where the sealer would refuse its + directory. Every one of these refuses before any GnuPG call. The gate reads the caller's scope once + and encrypts its own copy of the container, so neither can change midway; `encryptOnce` then runs + unchanged, on that copy and the caller's destination. - **The release binding receipt v1.** The public `release-binding-receipt.json` beside the ciphertext, the name the interim guard already keys on: @@ -713,10 +718,13 @@ refuses every artifact. Its review and acceptance are recorded in the and `release-scope.test.mjs` join `test:acceptance-tools`, as the 9b.1 tests do; the custody test needs GnuPG ≥ 2.2.8. `release-scope.test.mjs` pins the separation statically: - no product or tooling code reaches a test fixture, and only the loader reads the fixture pins; - - only the release gate imports `encryptOnce`, by any specifier that resolves to it; + - only the release gate imports `encryptOnce`, by any quoted specifier that resolves to it, in + source, workflow and package-script text, percent-escapes, queries, fragments and bare names + included; - the loader is never called with an argument, and its test seam appears in no product or tooling file (permanent controls); - - the receipt module's import closure starts no process, by import or otherwise; + - the receipt module's import closure holds only its own files and listed Node builtins, with no + package, process module, `require`-like call or non-literal dynamic import; - the interim guard still refuses a complete, valid synthetic candidate; - until Step 9d no product path or workflow names a release module, and until Wave 4b no recipient directory is committed. diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index d862ae5f..ec4b3cca 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -873,8 +873,8 @@ The release modules are library code with synthetic controls only: nothing outsi - [ ] **An orphaned ciphertext can never be sealed.** *Low; a named limit.* Only a record the gate returned in the same process seals, so a ciphertext whose seal never succeeds before its process exits — after a crash, or a refusal on an input the gate does not check, such as a malformed origin or an entry added to the - destination — is named by no record. The gate refuses an unresolvable destination and a missing or different - tarball before encrypting, which removes the likeliest causes. The candidate then needs manual recovery + destination after encryption — is named by no record. The gate refuses a linked or unresolvable destination, + any entry but the tarball and a different tarball before encrypting, which removes the likeliest causes. The candidate then needs manual recovery (packaging.md's missing-asset state); it is never re-encrypted. **Re-check:** in Step 9c's reservation states, classify such a directory as incomplete and route it to manual recovery. *(low · tools/cli-package/release-custody.mjs)* @@ -896,6 +896,12 @@ The release modules are library code with synthetic controls only: nothing outsi history naming it would load. The loader reads no key material (it makes no GnuPG call), and encrypting to it would still need its armored key in the committed recipient file. **Re-check:** list every committed fixture fingerprint in the denylist when the fixtures are next generated. *(low · tools/cli-package/release-recipient.mjs)* +- [ ] **The release entry points accept the test ports.** *Low; a named limit (round 1, SIT-R1-5).* + `encryptCandidate` forwards `encryptOnce`'s `run` and `gpg` ports and `sealReleaseBinding` takes a `write` + port, in production scope too, so an in-process caller could replace GnuPG and defeat its checks. Like the + brand, the ports guard against an honest mistake, not hostile code in the same process. **Re-check:** in Step + 9d, call both without ports from the release job and add a permanent control that no product call passes + them. *(low · tools/cli-package/release-custody.mjs)* - [ ] **The release modules do not run on Windows.** *Low; a named limit.* Like the 9b.1 tests, the release tests run only in `test:acceptance-tools`, which no Windows job runs; `encryptCandidate` refuses there through `encryptOnce`'s `PRIVATE_ROOT`, while reuse is platform-neutral but unexercised. **Re-check:** with @@ -2801,7 +2807,9 @@ future test cannot silently re-acquire it. recipient directory is committed. Any expiry margin is passed as the `now` the custody check uses. The loader sees one revision of `pins.json` and cannot detect a removed or rewritten entry, so Wave 4b adds a control that every committed revision keeps the previous one's entries, with only `current` becoming - `retired` (rigor pass, CLAIM-9B2-3). + `retired` (rigor pass, CLAIM-9B2-3). With the files committed, Wave 4b also replaces the absence test with a + load through `loadProductionRecipient()` itself, so the production path's own denylist and root wiring are + exercised: in Wave 3 the absent directory refuses before either is read (round 1, TEST-9B2-R1-4). - [ ] **Scheduled → Wave 5a: alt-screen word wrapping (#61).** The maintainer approved retaining character-wrap replacement with render-v2/D18 and picker diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 2023ad45..cba77008 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -10399,11 +10399,14 @@ Step 9b.2 is implemented on `development` in the commits from `84b8e455` through `release-custody` and `release-scope`. They include a production round trip, in which an ephemeral key under a rotated history decrypts to the container whose manifest the receipt binds, and fault injection after the link and after encryption. -- **Docs.** packaging.md's Step 9b.2 boundary is the schema's canonical home. deferred-tasks.md records three named - limits (an orphaned ciphertext, no recipient re-check on reuse, Windows not exercised), the Step 9d handoffs and a - Wave 4b clause on the recipient's committed form. - -**Sweeps before the rigor pass.** A generated sweep mutated every one-line refusal, condition clause and comparison of the three modules: +- **Docs.** packaging.md's Step 9b.2 boundary is the schema's canonical home. deferred-tasks.md records the named + limits (as of round 1: an orphaned ciphertext, no recipient re-check on reuse, a sibling left after the link, the + fixture denylist's coverage, the test ports and Windows not exercised), the Step 9d handoffs and a Wave 4b clause on + the recipient's committed form. + +**Sweeps before the rigor pass.** A generated sweep mutated every refusal statement, every guarded one-line refusal, +every clause of a multi-line condition and every comparison of the three modules; an inner clause of a one-line +compound condition is not mutated on its own: 131 mutants, 119 killed at first. Five survivors were real gaps and gained cases: - a `pending` status beside the current one; - a one-entry inputs map; @@ -10460,4 +10463,49 @@ removal the loader's wrapper maps to the same code, the inputs comparator, and t named mutants, 28 are killed. The survivor, the synthetic recipient's snapshot, is equivalent: synthetic scope admits any recipient, and `encryptOnce` checks the file it reads against the pins it parsed. -The full CI and the Ubuntu 2.4.4 evidence follow before the first round. +**Author evidence at `91d08a65`.** The full `pnpm run ci` passed plain and alone (exit 0): `test:acceptance-tools` +1,019 of 1,019, all 25 product map controls, and every causal-control family. On Ubuntu 24.04 with GnuPG 2.4.4 (in +Docker, a fresh clone, a non-root user), the 85 tests of the 9b.1 custody files and the four release files ran: 84 +passed and the case-insensitive-volume case skipped. + +**Round 1 at `91d08a65` — NONCLEAN.** Five lenses and a critic ran, every finding adversarially verified. Three +mediums, corrected: +- **The rigor pass's destination fix dropped a 9b.1 refusal.** Resolving the destination in the gate and handing + `encryptOnce` the resolved path let a linked destination through, which `encryptOnce` alone refuses. The gate now + refuses a link itself (`PLACEMENT`, before anything is read through it) and passes `encryptOnce` the caller's + destination, so its refusals apply unchanged (PLAN-9B2-R1-1, COR-9B2-R1-1, SIT-R1-1). +- **A stray entry led to an unsealable ciphertext.** The gate checked the tarball but not the destination's other + entries, which the sealer refuses. It now requires exactly the tarball: an existing ciphertext refuses + `ALREADY_ENCRYPTED`, and anything else, including a tarball name in another case, refuses `CANDIDATE_ENTRIES`, all + before any GnuPG call (SIT-R1-2, PLAN-9B2-R1-2, COR-9B2-R1-2). +- **A bare `custody.mjs` specifier escaped the import control.** The control now reads every quoted literal, nested + ones included, decodes percent-escapes, ignores a query or fragment, and resolves a bare name as Node resolves + `new URL(…, import.meta.url)`; package scripts join its scan (INT-9B2-R1-A, TEST-9B2-R1-7, COR-9B2-R1-4, + PLAN-9B2-R1-3). + +Lows corrected: +- the validator reads each field once, and the public validator, parser and reuse refuse a malformed call or a + throwing accessor with a fixed code (SIT-R1-6, COR-9B2-R1-3); a vanished asset refuses `CANDIDATE_ASSET` + (COR-9B2-R1-5); +- `pins.json` must be its canonical serialization, so a duplicated key refuses, and the checks that this subsumes are + removed (CRIT-9B2-R1-2); +- the closure control is an allowlist of its own files and listed builtins, and refuses a non-literal dynamic import + and `process.execve` (SIT-R1-3, INT-9B2-R1-B); +- the fixture control covers every `*-fixture.mjs` name (SIT-R1-4); +- the overhead constant's parity reads custody.mjs's own declaration (TEST-9B2-R1-6); +- new tests cover anchored patterns, a non-string whose string form matches, the forwarding of the clock and the + GnuPG installation, the reuse snapshot and a directory under the tarball's name (TEST-9B2-R1-2, TEST-9B2-R1-3, + TEST-9B2-R1-5, CRIT-9B2-R1-1); +- wording: this checkpoint's named-limit list and its sweep description (INT-9B2-R1-C, CRIT-9B2-R1-1). + +Recorded in deferred-tasks.md: the release entry points accept the test ports (SIT-R1-5), and the production loader's +own denylist wiring is exercised only once Wave 4b commits the files (TEST-9B2-R1-4). The oversized-receipt case of +TEST-9B2-R1-1 stays as it is: no canonical receipt reaches the 64 KiB bound, which only spares the parser an +oversized input. + +**Sweeps at the round 1 corrections.** The generated sweep ran 131 mutants and killed 127. The four survivors are +equivalent: the fixture entry's record check and the recipient files' read refusal, whose removal leads to a host error +that the loader's wrapper maps to the same code, the inputs comparator, and the receipt bound's `>=` flip. Of 38 named +mutants, 36 are killed. The two survivors are equivalent: passing `encryptOnce` the resolved destination, because the +gate's own `lstat` refuses every link first and `encryptOnce` resolves the path itself, and the synthetic recipient's +snapshot, as above. diff --git a/tools/cli-package/release-binding.mjs b/tools/cli-package/release-binding.mjs index 0474c84e..11dac155 100644 --- a/tools/cli-package/release-binding.mjs +++ b/tools/cli-package/release-binding.mjs @@ -119,12 +119,13 @@ function scopeOf(expected) { * @param {number} minimum @param {number} maximum */ function asset(value, version, name, minimum, maximum) { - const entry = exact(value, ['name', 'bytes', 'sha256']); - if (entry.name !== name(version)) refuseCustody('BINDING_NAME'); + // Each field is read once: the value checked is the value returned. + const { name: given, bytes, sha256: digest } = exact(value, ['name', 'bytes', 'sha256']); + if (given !== name(version)) refuseCustody('BINDING_NAME'); return Object.freeze({ - name: /** @type {string} */ (entry.name), - bytes: size(entry.bytes, minimum, maximum), - sha256: string(entry.sha256, SHA256), + name: /** @type {string} */ (given), + bytes: size(bytes, minimum, maximum), + sha256: string(digest, SHA256), }); } @@ -136,7 +137,21 @@ function asset(value, version, name, minimum, maximum) { * @param {Expected} expected */ export function validateReleaseBinding(value, expected) { - return validate(value, scopeOf(expected)); + return bounded(() => validate(value, scopeOf(expected))); +} + +/** + * Run `work` behind the fixed-code boundary: a host error's message, or a throwing accessor's, never + * leaves; only custody codes do. + * @template T @param {() => T} work @returns {T} + */ +function bounded(work) { + try { + return work(); + } catch (error) { + if (error instanceof CustodyError) throw error; + return refuseCustody('UNEXPECTED'); + } } /** @param {unknown} value @param {ReturnType} scope */ @@ -146,12 +161,12 @@ function validate(value, scope) { } const receipt = exact(value, KEYS); const packageVersion = string(receipt.packageVersion, PACKAGE_VERSION, 100); - const sourceRecord = exact(receipt.source, ['commit', 'tree', 'dirty']); - if (typeof sourceRecord.dirty !== 'boolean') refuseCustody('BINDING_SHAPE'); + const { commit, tree, dirty } = exact(receipt.source, ['commit', 'tree', 'dirty']); + if (typeof dirty !== 'boolean') refuseCustody('BINDING_SHAPE'); const source = Object.freeze({ - commit: string(sourceRecord.commit, GIT_OBJECT), - tree: string(sourceRecord.tree, GIT_OBJECT), - dirty: sourceRecord.dirty, + commit: string(commit, GIT_OBJECT), + tree: string(tree, GIT_OBJECT), + dirty, }); if (packageVersion !== scope.packageVersion) refuseCustody('BINDING_CANDIDATE'); if (scope.scope === 'production') { @@ -277,7 +292,7 @@ export const serializeReleaseBinding = (receipt) => * @param {Buffer} bytes @param {Expected} expected */ export function parseReleaseBinding(bytes, expected) { - return parse(bytes, scopeOf(expected)); + return bounded(() => parse(bytes, scopeOf(expected))); } /** @param {Buffer} bytes @param {ReturnType} scope */ @@ -300,14 +315,11 @@ function parse(bytes, scope) { * verified bytes, never a path to open again; writes, renames and removes nothing. * @param {{directory: string, expected: Expected}} options */ -export function verifyRetainedCandidate({ directory, expected }) { - try { +export function verifyRetainedCandidate(options) { + return bounded(() => { + const { directory, expected } = isRecord(options) ? options : {}; return verify(directory, expected); - } catch (error) { - // A host error's message names a path: only fixed custody codes leave. - if (error instanceof CustodyError) throw error; - return refuseCustody('UNEXPECTED'); - } + }); } /** @param {string} directory @param {Expected} expected */ @@ -333,7 +345,12 @@ function verify(directory, expected) { /** @param {{name: string, bytes: number, sha256: string}} entry */ const retained = (entry) => { const path = join(directory, entry.name); - const info = lstatSync(path); + let info; + try { + info = lstatSync(path); + } catch { + return refuseCustody('CANDIDATE_ASSET'); + } if (!info.isFile()) refuseCustody('CANDIDATE_ASSET'); if (info.size !== entry.bytes) refuseCustody('CANDIDATE_SIZE'); let data; diff --git a/tools/cli-package/release-binding.test.mjs b/tools/cli-package/release-binding.test.mjs index 23be5901..b85475a8 100644 --- a/tools/cli-package/release-binding.test.mjs +++ b/tools/cli-package/release-binding.test.mjs @@ -60,7 +60,10 @@ function production(...recipients) { const root = fresh('recipient'); mkdirSync(root); writeFileSync(join(root, 'recipient.asc'), 'key'); - writeFileSync(join(root, 'pins.json'), JSON.stringify({ version: 1, recipients })); + writeFileSync( + join(root, 'pins.json'), + `${JSON.stringify({ version: 1, recipients }, null, 2)}\n`, + ); return loadRecipientForTests({ root }); } const ROTATED = production({ ...A, status: 'retired' }, { ...B, status: 'current' }); @@ -261,6 +264,23 @@ test('every value has its exact format', () => { (copy) => (copy.origin.runAttempt = '01'), (copy) => (copy.origin.runAttempt = '1234567'), (copy) => (copy.origin.repository = `${'a'.repeat(100)}/${'b'.repeat(100)}`), + // Every pattern is anchored at both ends: a prefix before a valid value refuses. + (copy) => (copy.source.commit = `x${copy.source.commit}`), + (copy) => (copy.source.tree = `x${copy.source.tree}`), + (copy) => (copy.tarball.sha256 = `x${copy.tarball.sha256}`), + (copy) => (copy.recipient.primary = `X${copy.recipient.primary}`), + (copy) => (copy.toolchain.node = `x${copy.toolchain.node}`), + (copy) => (copy.toolchain.typescript = `x${copy.toolchain.typescript}`), + (copy) => (copy.toolchain.gnupg = `x${copy.toolchain.gnupg}`), + (copy) => (copy.target = `x${copy.target}`), + (copy) => (copy.origin.repository = ` ${copy.origin.repository}`), + (copy) => (copy.origin.workflow = `\u0007${copy.origin.workflow}`), + (copy) => (copy.origin.runId = `x${copy.origin.runId}`), + (copy) => (copy.origin.runAttempt = `x${copy.origin.runAttempt}`), + (copy) => (copy.inputs = { 'package.json': `x${'a'.repeat(64)}` }), + // A value whose string form matches is still not a string. + (copy) => (copy.toolchain.node = [copy.toolchain.node]), + (copy) => (copy.source.commit = [copy.source.commit]), (copy) => (copy.origin.workflow = 'w'.repeat(201)), (copy) => (copy.origin.runId = '1'.repeat(21)), // Integer-like names would sort ahead of every path in the canonical bytes. @@ -445,6 +465,8 @@ test('only the exact canonical bytes parse', () => { })(), ], ['not JSON', Buffer.from('receipt')], + // Also refused as non-canonical: no canonical receipt reaches the 64 KiB bound, which only spares the + // parser an oversized input. ['too large', Buffer.concat([bytes, Buffer.alloc(64 * 1024 + 1 - bytes.length, 0x20)])], [ 'unsorted inputs', @@ -748,6 +770,92 @@ test( }, ); +test('each receipt field is read once, and a throwing accessor refuses with a fixed code', () => { + // A name checked on its first read is the name returned, never a later value. + let reads = 0; + const value = receipt(); + Object.defineProperty(value.tarball, 'name', { + enumerable: true, + get() { + reads += 1; + return reads === 1 ? tarballName(VERSION) : '/private/host/path'; + }, + }); + assert.equal(validateReleaseBinding(value, synthetic).tarball.name, tarballName(VERSION)); + assert.equal(reads, 1); + // A clean source checked once stays clean in the receipt returned. + let dirtyReads = 0; + const clean = productionReceipt(); + Object.defineProperty(clean.source, 'dirty', { + enumerable: true, + get() { + dirtyReads += 1; + return dirtyReads !== 1; + }, + }); + assert.equal(validateReleaseBinding(clean, productionExpected()).source.dirty, false); + assert.equal(dirtyReads, 1); + const throwing = receipt(); + Object.defineProperty(throwing, 'target', { + enumerable: true, + get() { + throw new Error(`at ${scratch}`); + }, + }); + assert.throws(() => validateReleaseBinding(throwing, synthetic), refuses('UNEXPECTED')); + const expectation = { + ...synthetic, + get pins() { + throw new Error(`at ${scratch}`); + }, + }; + assert.throws( + () => + parseReleaseBinding( + serializeReleaseBinding(validateReleaseBinding(receipt(), synthetic)), + expectation, + ), + refuses('UNEXPECTED'), + ); +}); + +test('reuse reads its expectation once, and a vanished asset refuses as that asset', () => { + const directory = candidate(); + let reads = 0; + const flipping = { + scope: 'synthetic', + pins: PINS, + get packageVersion() { + reads += 1; + return reads === 1 ? VERSION : '0.1.2'; + }, + }; + verifyRetainedCandidate({ directory, expected: flipping }); + assert.equal(reads, 1); + // An asset that disappears after the listing refuses as that asset, never a host error. + const fs = createRequire(import.meta.url)('node:fs'); + const lstat = fs.lstatSync; + const path = join(directory, tarballName(VERSION)); + fs.lstatSync = (target, ...rest) => { + if (target === path) throw Object.assign(new Error(`ENOENT: ${path}`), { code: 'ENOENT' }); + return lstat(target, ...rest); + }; + syncBuiltinESMExports(); + try { + assert.throws( + () => verifyRetainedCandidate({ directory, expected: synthetic }), + refuses('CANDIDATE_ASSET'), + ); + } finally { + fs.lstatSync = lstat; + syncBuiltinESMExports(); + } + // A malformed call refuses with a fixed code. + for (const options of [undefined, null, 'options']) { + assert.throws(() => verifyRetainedCandidate(options), refuses('BINDING_SCOPE')); + } +}); + test('reuse needs a valid expectation, and a host failure refuses with a fixed code and no path', () => { const directory = candidate(); for (const [code, expected] of [ diff --git a/tools/cli-package/release-custody.mjs b/tools/cli-package/release-custody.mjs index bb628d67..564f279e 100644 --- a/tools/cli-package/release-custody.mjs +++ b/tools/cli-package/release-custody.mjs @@ -29,6 +29,7 @@ import { readOrdinaryFile, sha256 } from './files.mjs'; import { RELEASE_BINDING_FILE, RELEASE_BINDING_KIND, + ciphertextFileName, serializeReleaseBinding, tarballName, validateReleaseBinding, @@ -64,24 +65,21 @@ function present(path) { /** * Encrypt a candidate's container once for its scope. Every refusal here comes before any GnuPG call; - * `encryptOnce`'s own refusals follow unchanged. + * `encryptOnce` then runs on the caller's destination, so its own refusals follow unchanged. * @param {{scope: Scope, container: Buffer, destination: string, privateRoot: string, now?: number, * run?: typeof import('./process.mjs').runCommand, gpg?: import('./gnupg.mjs').GnupgInstallation}} * options * @returns {Promise} */ -export async function encryptCandidate({ - scope, - container, - destination, - privateRoot, - now, - run, - gpg, -}) { +export async function encryptCandidate(options) { let admitted; + let ports; try { + const { scope, container, destination, privateRoot, now, run, gpg } = isRecord(options) + ? options + : {}; admitted = admit(scope, container, destination); + ports = { destination, privateRoot, now, run, gpg }; } catch (error) { // A host error's message names a path: only fixed custody codes leave. if (error instanceof CustodyError) throw error; @@ -89,13 +87,9 @@ export async function encryptCandidate({ } // The admitted copy is the one encrypted: the caller's buffer may change, it may not. const record = await encryptOnce({ + ...ports, container: admitted.container, recipient: admitted.recipient, - destination: admitted.destination, - privateRoot, - now, - run, - gpg, }); records.set(record, { kind: admitted.kind, @@ -107,8 +101,8 @@ export async function encryptCandidate({ } /** - * Admit one encryption, reading the caller's scope and container once: the scope's rules, an unsealed - * destination and the candidate's tarball, all before any GnuPG call. + * Admit one encryption, reading the caller's scope and container once: the scope's rules, then a plain, + * unsealed destination that holds exactly the candidate's tarball, all before any GnuPG call. * @param {unknown} scope @param {unknown} container @param {string} destination */ function admit(scope, container, destination) { @@ -129,21 +123,27 @@ function admit(scope, container, destination) { recipient = { file: given.file, pins: given.pins }; } if (present(join(destination, RELEASE_BINDING_FILE))) refuseCustody('ALREADY_SEALED'); - // Resolved now, so nothing after the ciphertext is published can fail on the destination. + // A plain directory, as encryptOnce requires, resolved now so that nothing after the ciphertext is + // published can fail on it. let resolved; try { + if (!lstatSync(destination).isDirectory()) refuseCustody('PLACEMENT'); resolved = realpathSync(destination); - } catch { + } catch (error) { + if (error instanceof CustodyError) throw error; refuseCustody('PLACEMENT'); } - // The receipt binds the tarball the manifest names; a missing or different one would leave an - // unsealable ciphertext, so it refuses first. + // The sealer admits exactly the tarball and the ciphertext: a destination holding anything else, or a + // missing or different tarball, would leave an unsealable ciphertext, so it refuses first. + const tarballFile = tarballName(manifest.packageVersion); + const entries = readdirSync(resolved); + if (entries.includes(ciphertextFileName(manifest.packageVersion))) { + refuseCustody('ALREADY_ENCRYPTED'); + } + if (!isDeepStrictEqual(entries, [tarballFile])) refuseCustody('CANDIDATE_ENTRIES'); let tarball; try { - tarball = readOrdinaryFile( - join(resolved, tarballName(manifest.packageVersion)), - ARCHIVE_LIMITS.compressedBytes, - ); + tarball = readOrdinaryFile(join(resolved, tarballFile), ARCHIVE_LIMITS.compressedBytes); } catch { refuseCustody('TARBALL_DRIFT'); } diff --git a/tools/cli-package/release-custody.test.mjs b/tools/cli-package/release-custody.test.mjs index 1434f179..064e9dca 100644 --- a/tools/cli-package/release-custody.test.mjs +++ b/tools/cli-package/release-custody.test.mjs @@ -130,14 +130,44 @@ test('the gate refuses a wrong scope, recipient or candidate before any GnuPG ca } // A destination that cannot be resolved, or whose tarball is missing or another, refuses before any // call: a ciphertext encrypted into it could never be sealed. + // A link refuses before anything is read through it: its target's stray entry is never reached. + const linked = (destination) => { + const target = fresh('linked-destination-target'); + mkdirSync(target); + writeFileSync(join(target, tarballName(VERSION)), TARBALL); + writeFileSync(join(target, 'stray'), 'x'); + rmSync(destination, { recursive: true }); + symlinkSync(target, destination); + }; for (const [code, change] of [ ['PLACEMENT', (destination) => rmSync(destination, { recursive: true })], - ['TARBALL_DRIFT', (destination) => rmSync(join(destination, tarballName(VERSION)))], + [ + 'PLACEMENT', + (destination) => + posixModes ? linked(destination) : rmSync(destination, { recursive: true }), + ], + ['CANDIDATE_ENTRIES', (destination) => rmSync(join(destination, tarballName(VERSION)))], + [ + 'CANDIDATE_ENTRIES', + (destination) => writeFileSync(join(destination, 'package-test-receipt.json'), '{}'), + ], + [ + 'ALREADY_ENCRYPTED', + (destination) => + writeFileSync(join(destination, ciphertextFileName(VERSION)), 'earlier ciphertext'), + ], [ 'TARBALL_DRIFT', (destination) => writeFileSync(join(destination, tarballName(VERSION)), Buffer.from('another tarball')), ], + [ + 'TARBALL_DRIFT', + (destination) => { + rmSync(join(destination, tarballName(VERSION))); + mkdirSync(join(destination, tarballName(VERSION))); + }, + ], ]) { const { root, destination } = places(); change(destination); @@ -155,6 +185,28 @@ test('the gate refuses a wrong scope, recipient or candidate before any GnuPG ca ); assert.deepEqual(calls, [], code); } + // A tarball whose name differs only in case is not the tarball, even where the volume would open it. + { + const { root, destination } = places(); + rmSync(join(destination, tarballName(VERSION))); + writeFileSync(join(destination, tarballName(VERSION).toUpperCase()), TARBALL); + const { run, calls } = recording(); + await assert.rejects( + encryptCandidate({ + scope: synthetic, + container: built.synthetic, + destination, + privateRoot: root, + run, + }), + refuses('CANDIDATE_ENTRIES'), + ); + assert.deepEqual(calls, []); + } + // A malformed call refuses with a fixed code, never a host error. + for (const options of [undefined, null, 'options']) { + await assert.rejects(encryptCandidate(options), refuses('SCOPE')); + } // The scope is read once: a kind that changes after its first read cannot skip the brand check. { let reads = 0; @@ -519,6 +571,33 @@ test("the caller's container changing during encryption changes nothing encrypte sealReleaseBinding(record, { origin: origin(SYNTHETIC_SOURCE.commit) }); }); +test('the gate forwards the clock and the GnuPG installation to encryptOnce', async () => { + // A clock before the fixture key's creation makes its subkey one created in the future. + const early = places(); + await assert.rejects( + encryptCandidate({ + scope: synthetic, + container: built.synthetic, + destination: early.destination, + privateRoot: early.root, + now: 1, + }), + refuses('RECIPIENT_UNUSABLE'), + ); + const missing = places(); + const absent = { command: join(fresh('no-gpg'), 'gpg'), prefix: [] }; + await assert.rejects( + encryptCandidate({ + scope: synthetic, + container: built.synthetic, + destination: missing.destination, + privateRoot: missing.root, + gpg: { ...absent, gpgconf: absent }, + }), + refuses('GPG_UNAVAILABLE'), + ); +}); + test('a production candidate round-trips: the receipt binds what the ciphertext decrypts to', async () => { const ephemeral = ephemeralKey(); // A rotated history: an earlier recipient retired, the ephemeral key current. diff --git a/tools/cli-package/release-fixture.mjs b/tools/cli-package/release-fixture.mjs index a769ded2..a2695d94 100644 --- a/tools/cli-package/release-fixture.mjs +++ b/tools/cli-package/release-fixture.mjs @@ -77,6 +77,10 @@ export function recipientRoot(file, recipients) { const root = fresh('release-recipient'); mkdirSync(root); copyFileSync(file, join(root, 'recipient.asc')); - writeFileSync(join(root, 'pins.json'), JSON.stringify({ version: 1, recipients })); + // The loader admits only the canonical serialization. + writeFileSync( + join(root, 'pins.json'), + `${JSON.stringify({ version: 1, recipients }, null, 2)}\n`, + ); return root; } diff --git a/tools/cli-package/release-recipient.mjs b/tools/cli-package/release-recipient.mjs index 45f79563..6ab47496 100644 --- a/tools/cli-package/release-recipient.mjs +++ b/tools/cli-package/release-recipient.mjs @@ -7,6 +7,7 @@ * exist, so every production use refuses; Wave 4b commits the files. A loaded recipient is branded: only the * value this module returned counts as a production recipient, never a copy of it. */ +import { Buffer } from 'node:buffer'; import { lstatSync, readdirSync } from 'node:fs'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -58,27 +59,20 @@ function parseHistory(bytes) { } catch { return refuseCustody('PRODUCTION_RECIPIENT_INVALID'); } - if ( - !isRecord(value) || - !isDeepStrictEqual(Object.keys(value).sort(), ['recipients', 'version']) || - value.version !== 1 || - !Array.isArray(value.recipients) || - value.recipients.length < 1 || - value.recipients.length > MAX_RECIPIENTS - ) { - refuseCustody('PRODUCTION_RECIPIENT_INVALID'); - } - const history = value.recipients.map((entry) => { + // The canonical comparison below fixes the version and every key set; these checks make the history + // safe to build and state its rules. + const recipients = isRecord(value) && Array.isArray(value.recipients) ? value.recipients : []; + if (recipients.length > MAX_RECIPIENTS) refuseCustody('PRODUCTION_RECIPIENT_INVALID'); + const history = recipients.map((entry) => { + const { primary, subkey, status } = isRecord(entry) ? entry : {}; if ( - !isRecord(entry) || - !isDeepStrictEqual(Object.keys(entry).sort(), ['primary', 'status', 'subkey']) || - !isFingerprint(entry.primary) || - !isFingerprint(entry.subkey) || - (entry.status !== 'current' && entry.status !== 'retired') + !isFingerprint(primary) || + !isFingerprint(subkey) || + (status !== 'current' && status !== 'retired') ) { refuseCustody('PRODUCTION_RECIPIENT_INVALID'); } - return Object.freeze({ primary: entry.primary, subkey: entry.subkey, status: entry.status }); + return Object.freeze({ primary, subkey, status }); }); if ( history.filter((entry) => entry.status === 'current').length !== 1 || @@ -86,6 +80,13 @@ function parseHistory(bytes) { ) { refuseCustody('PRODUCTION_RECIPIENT_INVALID'); } + // Exactly its canonical serialization, so the file a reviewer reads is the history loaded: a + // duplicated key, which JSON.parse would resolve silently to its last value, or another order or + // layout refuses. + const canonical = Buffer.from( + `${JSON.stringify({ version: 1, recipients: history }, null, 2)}\n`, + ); + if (!canonical.equals(bytes)) refuseCustody('PRODUCTION_RECIPIENT_INVALID'); return Object.freeze(history); } diff --git a/tools/cli-package/release-recipient.test.mjs b/tools/cli-package/release-recipient.test.mjs index 36543a3f..d98df0da 100644 --- a/tools/cli-package/release-recipient.test.mjs +++ b/tools/cli-package/release-recipient.test.mjs @@ -40,7 +40,10 @@ function root({ const path = fresh('root'); mkdirSync(path); writeFileSync(join(path, 'recipient.asc'), key); - writeFileSync(join(path, 'pins.json'), typeof pins === 'string' ? pins : JSON.stringify(pins)); + writeFileSync( + join(path, 'pins.json'), + typeof pins === 'string' ? pins : `${JSON.stringify(pins, null, 2)}\n`, + ); return path; } const history = (...recipients) => ({ version: 1, recipients }); @@ -174,6 +177,17 @@ test('the pins history is exactly one current recipient among at most 32 distinc history({ primary: 'a'.repeat(40), subkey: A.subkey, status: 'current' }), history({ primary: [A.primary], subkey: A.subkey, status: 'current' }), history({ primary: A.primary, subkey: `${A.subkey}!`, status: 'current' }), + history({ primary: `X${A.primary}`, subkey: A.subkey, status: 'current' }), + // Only the canonical serialization: a compact layout, a missing final newline, another key order or a + // duplicated key, which JSON.parse would resolve silently to its last value. + JSON.stringify(history({ ...A, status: 'current' })), + JSON.stringify(history({ ...A, status: 'current' }), null, 2), + `${JSON.stringify({ recipients: [{ ...A, status: 'current' }], version: 1 }, null, 2)}\n`, + `${JSON.stringify(history({ status: 'current', ...A }), null, 2)}\n`, + `${JSON.stringify(history({ ...A, status: 'retired' }), null, 2)}\n`.replace( + '"status": "retired"', + '"status": "retired",\n "status": "current"', + ), history( current, ...Array.from({ length: 32 }, (_, index) => ({ diff --git a/tools/cli-package/release-scope.test.mjs b/tools/cli-package/release-scope.test.mjs index 7fe73a18..22ad2f4f 100644 --- a/tools/cli-package/release-scope.test.mjs +++ b/tools/cli-package/release-scope.test.mjs @@ -41,6 +41,9 @@ const tracked = spawnSync( '*.mts', '*.yml', '*.yaml', + // Package scripts can name a module too. + 'package.json', + '*/package.json', ], { cwd: repository, encoding: 'utf8' }, ) @@ -76,25 +79,32 @@ test('the scan sees the custody modules and leaves the tests out', () => { }); test('no product or tooling code reaches a test fixture, and only the loader reads the fixture pins', () => { - assert.deepEqual(naming(/(?:candidate|gpg|release)-fixture\.mjs/u), []); + assert.deepEqual(naming(/[A-Za-z0-9_-]+-fixture\.mjs/u), []); assert.deepEqual(naming(/custody-fixtures/u), ['tools/cli-package/release-recipient.mjs']); }); /** * Whether `text`, a file at repository-relative `path`, names repository-relative `target` by any quoted - * specifier that resolves to it: relative to the file, or to the repository root for a workflow (whose - * steps run there) and for a `tools/` specifier; a query suffix is ignored. + * literal that resolves to it, as Node would resolve a relative specifier or a `new URL(…, + * import.meta.url)`: relative to the file, or to the repository root for a workflow (whose steps run + * there), a package manifest's scripts and a `tools/` path. Percent-escapes are decoded and a query or + * fragment is ignored. * @param {string} path @param {string} text @param {string} target */ function resolvesTo(path, text, target) { - const name = posix.basename(target).replaceAll('.', '\\.'); - const literal = new RegExp(`['"\`]([^'"\`\\s]*(?:^|/)${name})(?:\\?[^'"\`\\s]*)?['"\`]`, 'gu'); - const root = /\.ya?ml$/u.test(path) ? '.' : posix.dirname(path); - for (const match of text.matchAll(literal)) { - const specifier = match[1]; - const base = specifier.startsWith('tools/') ? '.' : root; - const resolved = posix.normalize(posix.join(base, specifier)); - if (resolved === target) return true; + const root = /(?:\.ya?ml|package\.json)$/u.test(path) ? '.' : posix.dirname(path); + // Every run of non-space characters directly between two quotes; the closing quote may open the next + // run, so a literal nested inside another (a workflow's `node -e "import('…')"`) is found too. + for (const match of text.matchAll(/['"`]([^'"`\s]+)(?=['"`])/gu)) { + let specifier = match[1].replace(/[?#].*$/u, ''); + try { + specifier = decodeURIComponent(specifier); + } catch { + continue; + } + if (posix.basename(specifier) !== posix.basename(target)) continue; + const base = specifier.startsWith('tools/') || specifier.startsWith('./tools/') ? '.' : root; + if (posix.normalize(posix.join(base, specifier)) === target) return true; } return false; } @@ -105,7 +115,11 @@ test('the import match resolves every spelling, and only the real target', () => ['tools/cli-package/smoke/x.mjs', "import { encryptOnce } from '../custody.mjs';"], ['tools/x.mjs', "import { encryptOnce } from './cli-package/custody.mjs';"], ['tools/ci/x/y.mjs', 'const m = await import("../../cli-package/custody.mjs?v=1");'], + ['tools/cli-package/x.mjs', "import { encryptOnce } from './custody.mjs#fragment';"], + ['tools/cli-package/x.mjs', "import { encryptOnce } from './custody%2Emjs';"], + ['tools/cli-package/x.mjs', "const url = new URL('custody.mjs', import.meta.url);"], ['.github/workflows/x.yml', 'run: node -e "import(\'./tools/cli-package/custody.mjs\')"'], + ['package.json', '"x": "node -e \\"import(\'./tools/cli-package/custody.mjs\')\\""'], ['apps/cli/x.ts', "export * as custody from '../../tools/cli-package/custody.mjs';"], ['tools/cli-package/x.mjs', 'require(`./custody.mjs`)'], ]) { @@ -116,6 +130,7 @@ test('the import match resolves every spelling, and only the real target', () => ['tools/cli-package/x.mjs', "import { encryptCandidate } from './release-custody.mjs';"], ['tools/x.mjs', "import { x } from './custody.mjs';"], ['tools/cli-package/x.mjs', '// encryptOnce lives in custody.mjs'], + ['tools/cli-package/x.mjs', "import { x } from './custody.mjs.bak';"], ]) { assert.equal(resolvesTo(path, text, target), false, `${path}: ${text}`); } @@ -177,16 +192,26 @@ test("the binding receipt's closure starts no process, so reuse makes no GnuPG c 'release-recipient.mjs', 'ustar.mjs', ]); - for (const forbidden of ['child_process', 'worker_threads', 'cluster', 'module']) { - assert.equal(specifiers.includes(forbidden), false, forbidden); - assert.equal(specifiers.includes(`node:${forbidden}`), false, forbidden); + // Every specifier is one of those files or an allowed builtin: no package, no process module. + const builtins = [ + 'node:assert/strict', + 'node:buffer', + 'node:crypto', + 'node:fs', + 'node:path', + 'node:url', + 'node:util', + 'node:zlib', + ]; + for (const specifier of specifiers) { + assert.ok(specifier.startsWith('./') || builtins.includes(specifier), specifier); } - // Nor can a closure file reach a process without importing one. + // Nor can a closure file reach a process or a module without a literal import. for (const file of files) { const source = readFileSync(join(here, file), 'utf8'); assert.doesNotMatch( source, - /\b(?:getBuiltinModule|createRequire|process\.binding|process\.dlopen)\b|\brequire\s*\(/u, + /\b(?:getBuiltinModule|createRequire|process\.binding|process\.dlopen|process\.execve)\b|\brequire\s*\(|\bimport\(\s*[^'"\s]/u, file, ); } @@ -268,6 +293,14 @@ test('the constants the release modules restate equal their sources', () => { for (const version of ['0.1.1', '1.2.3-rc.1', '2.0.0+build.5']) { assert.equal(ciphertextFileName(version), ciphertextName(version)); } - assert.equal(CIPHERTEXT_BYTES, CONTAINER_LIMITS.totalBytes + 64 * 1024); + // custody.mjs keeps its overhead private: read it from the source, so both halves have a source. + const overhead = /^const CIPHERTEXT_OVERHEAD = (\d+) \* (\d+);$/mu.exec( + read('tools/cli-package/custody.mjs'), + ); + assert.ok(overhead, 'custody.mjs no longer declares CIPHERTEXT_OVERHEAD as a product'); + assert.equal( + CIPHERTEXT_BYTES, + CONTAINER_LIMITS.totalBytes + Number(overhead[1]) * Number(overhead[2]), + ); assert.deepEqual(SYNTHETIC_SOURCE, { commit: COMMIT, tree: TREE }); }); From 463a047c93c2cdbccdda6d13244010b78c2b294c Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 26 Oct 2026 03:43:52 +0300 Subject: [PATCH 85/97] fix(repo): address the second Step 9b.2 review round Round 2 at 5cda6459 reported two mediums: - the import control resolved every package manifest against the repository root, but pnpm runs a package's scripts in its directory, so a nested script could reach custody.mjs unseen; the control now refuses any quoted reference, outside the gate, to a file named custody.mjs, the one tracked file of that name, with separators, queries, fragments, percent-escapes and case normalized; - a trailing separator bypassed the gate's link refusal, because lstat('link/') follows the link; the destination and the private root must now be normalized absolute paths, and the destination is checked before it is probed for a receipt. Lows corrected: the sealer reads its options inside its fixed-code boundary; the validator reads the origin and inputs once; a dynamic import in the receipt's closure must be one string literal; a recipient root whose parent is a file is never absent; the 9d handoff widens all three directory admissions together. One low is recorded: encryptOnce called directly still follows a trailing-separator link. Refs: #46, ADR-0111 Co-Authored-By: Claude Opus 5.5 --- docs/reference/cli/packaging.md | 17 +-- docs/roadmap/deferred-tasks.md | 11 +- .../phase-2.5.5-hardening-and-remediation.md | 31 +++++ tools/cli-package/release-binding.mjs | 11 +- tools/cli-package/release-custody.mjs | 19 ++-- tools/cli-package/release-custody.test.mjs | 65 ++++++++++- tools/cli-package/release-recipient.test.mjs | 7 ++ tools/cli-package/release-scope.test.mjs | 107 ++++++++++++------ 8 files changed, 207 insertions(+), 61 deletions(-) diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 11cac1bf..700c88b9 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -652,9 +652,11 @@ refuses every artifact. Its review and acceptance are recorded in the loaded recipient and a clean, non-synthetic candidate, and encrypts to the recipient's current pins (`RECIPIENT_SOURCE`, `DIRTY_SOURCE`, `SYNTHETIC_SOURCE`). It admits synthetic scope only with a synthetic candidate (`FIXTURE_SCOPE`), so neither a fixture key nor the per-run round-trip key can - encrypt a product staging. An unknown scope refuses `SCOPE`. The destination must be a plain, - resolvable directory (`PLACEMENT`; a link refuses, as `encryptOnce` requires) holding exactly the - tarball the manifest names: a receipt there refuses `ALREADY_SEALED`, the ciphertext + encrypt a product staging. An unknown scope refuses `SCOPE`. The destination and the private root + must be normalized absolute paths (`PLACEMENT`, `PRIVATE_ROOT`), so that a trailing separator, `.` or + `..` cannot make a link look like a directory. The destination must then be a plain, resolvable + directory (`PLACEMENT`; a link refuses before anything is read through it, as `encryptOnce` requires) + holding exactly the tarball the manifest names: a receipt there refuses `ALREADY_SEALED`, the ciphertext `ALREADY_ENCRYPTED`, any other entry, or a tarball named otherwise, `CANDIDATE_ENTRIES`, and a different tarball `TARBALL_DRIFT`. A ciphertext is therefore never encrypted where the sealer would refuse its directory. Every one of these refuses before any GnuPG call. The gate reads the caller's scope once @@ -718,13 +720,14 @@ refuses every artifact. Its review and acceptance are recorded in the and `release-scope.test.mjs` join `test:acceptance-tools`, as the 9b.1 tests do; the custody test needs GnuPG ≥ 2.2.8. `release-scope.test.mjs` pins the separation statically: - no product or tooling code reaches a test fixture, and only the loader reads the fixture pins; - - only the release gate imports `encryptOnce`, by any quoted specifier that resolves to it, in - source, workflow and package-script text, percent-escapes, queries, fragments and bare names - included; + - only the release gate refers to `custody.mjs`, the one tracked file of that name, by any quoted + literal in source, workflow and package-script text, whatever directory it would resolve from: + separators, queries, fragments, percent-escapes and case are normalized; - the loader is never called with an argument, and its test seam appears in no product or tooling file (permanent controls); - the receipt module's import closure holds only its own files and listed Node builtins, with no - package, process module, `require`-like call or non-literal dynamic import; + package, process module or `require`-like call, and every dynamic import takes exactly one string + literal; - the interim guard still refuses a complete, valid synthetic candidate; - until Step 9d no product path or workflow names a release module, and until Wave 4b no recipient directory is committed. diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index ec4b3cca..4714da5d 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -896,6 +896,12 @@ The release modules are library code with synthetic controls only: nothing outsi history naming it would load. The loader reads no key material (it makes no GnuPG call), and encrypting to it would still need its armored key in the committed recipient file. **Re-check:** list every committed fixture fingerprint in the denylist when the fixtures are next generated. *(low · tools/cli-package/release-recipient.mjs)* +- [ ] **`encryptOnce` called directly follows a link spelled with a trailing separator.** *Low; a named limit + (round 2, COR-R2-2).* 9b.1's `plainDirectory` and `privateDirectory` lstat the path as given, and `lstat('link/')` + follows the link, so a direct caller of `encryptOnce` can encrypt into a linked destination or use a linked + private root. The release gate requires both paths in normalized absolute form and refuses a link itself, so + no release path reaches this; nothing outside the gate and the tests calls `encryptOnce`. **Re-check:** on the + next change to `custody.mjs`, require normalized absolute paths there too. *(low · tools/cli-package/custody.mjs)* - [ ] **The release entry points accept the test ports.** *Low; a named limit (round 1, SIT-R1-5).* `encryptCandidate` forwards `encryptOnce`'s `run` and `gpg` ports and `sealReleaseBinding` takes a `write` port, in production scope too, so an in-process caller could replace GnuPG and defeat its checks. Like the @@ -911,8 +917,9 @@ The release modules are library code with synthetic controls only: nothing outsi and attempt (on reuse, the original run) and `mapCheck` to a successful `ci` run with the product map controls at `source.commit`; derive release smoke's public inventory and package metadata from the digest-verified tarball with the package-test inspector; check the production recipient before any - reservation, so a recipient refusal never leaves an incomplete reservation; widen the sealer's directory - admission if the package-test receipt travels beside the tarball; replace the interim guard; then update + reservation, so a recipient refusal never leaves an incomplete reservation; if the package-test receipt + travels beside the tarball, widen the gate's, the sealer's and reuse's directory admission together, since all + three admit exact entries (round 2, CRIT-9B2-R2-1); replace the interim guard; then update `release-scope.test.mjs`'s interim control that no product path or workflow names a release module, keeping its permanent controls (the loader takes no argument and its test seam stays out of product code; only the gate imports `encryptOnce`); and include `tools/cli-package/custody-fixtures/pins.json` in every checkout diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index cba77008..a7d9c733 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -10509,3 +10509,34 @@ that the loader's wrapper maps to the same code, the inputs comparator, and the mutants, 36 are killed. The two survivors are equivalent: passing `encryptOnce` the resolved destination, because the gate's own `lstat` refuses every link first and `encryptOnce` resolves the path itself, and the synthetic recipient's snapshot, as above. + +**Author evidence at `5cda6459`.** The full `pnpm run ci` passed plain and alone (exit 0): `test:acceptance-tools` +1,022 of 1,022, all 25 product map controls, and every causal-control family. On Ubuntu 24.04 with GnuPG 2.4.4, the +88 tests ran: 87 passed and the case-insensitive-volume case skipped. + +**Round 2 at `5cda6459` — NONCLEAN.** Five lenses and a critic ran, every finding adversarially verified. Two +mediums, corrected: +- **A nested package script escaped the import control.** pnpm runs a package's scripts in that package's directory, + but the control resolved every manifest against the repository root, so a script in `apps/cli/package.json` could + reach `custody.mjs` unseen; a workflow step's working directory left the same gap. No single base is right, so the + control now refuses any quoted reference, outside the gate, to a file named `custody.mjs`, the one tracked file of + that name, with separators, queries, fragments, percent-escapes and case normalized (PLAN-9B2-R2-1, COR-R2-1, + T-R2-3). +- **A trailing separator bypassed the link refusal.** `lstat('link/')` follows the link, so a linked destination + spelled with a trailing `/` or `/.` was admitted. The gate now requires the destination and the private root in + normalized absolute form, and checks the destination before probing it for a receipt, so a link refuses + `PLACEMENT` before anything is read through it (COR-R2-2, SIT2-R2-1, PLAN-9B2-R2-2, INT-9B2-R2-1). + +Lows corrected: +- the sealer reads its options inside its fixed-code boundary (SIT2-R2-2, COR-R2-4, T-R2-1); +- the validator reads the origin and the inputs once (COR-R2-5); +- a dynamic import in the receipt's closure must be exactly one string literal, a rule with its own self-test + (COR-R2-3); +- a recipient root whose parent is a file refuses `PRODUCTION_RECIPIENT_INVALID`, never absent (T-R2-2); +- the Step 9d handoff widens the gate's, the sealer's and reuse's directory admission together (CRIT-9B2-R2-1). + +Recorded in deferred-tasks.md: `encryptOnce` called directly still follows a link spelled with a trailing separator. +Nothing outside the gate and the tests calls it, and the gate requires normalized paths (the 9b.1 half of COR-R2-2). + +**Sweeps at the round 2 corrections.** The generated sweep ran 135 mutants and killed 131; the four survivors are the +equivalents named above. Of 40 named mutants, 38 are killed; the two survivors are the equivalents named above. diff --git a/tools/cli-package/release-binding.mjs b/tools/cli-package/release-binding.mjs index 11dac155..dd7ed41a 100644 --- a/tools/cli-package/release-binding.mjs +++ b/tools/cli-package/release-binding.mjs @@ -224,8 +224,9 @@ function validate(value, scope) { }); const target = string(receipt.target, /^node\d{2,3}$/u, 8); - if (!isRecord(receipt.inputs)) refuseCustody('BINDING_SHAPE'); - const entries = Object.entries(receipt.inputs); + const { inputs: inputsRecord, origin: originValue } = receipt; + if (!isRecord(inputsRecord)) refuseCustody('BINDING_SHAPE'); + const entries = Object.entries(inputsRecord); if (entries.length < 1 || entries.length > MAX_INPUTS) refuseCustody('BINDING_SHAPE'); const inputs = Object.freeze( Object.fromEntries( @@ -241,10 +242,8 @@ function validate(value, scope) { const mapCheck = exact(receipt.mapCheck, ['status']); if (mapCheck.status !== 'complete') refuseCustody('BINDING_MAP_CHECK'); - if (!isRecord(receipt.origin) || receipt.origin.kind !== 'github') { - refuseCustody('BINDING_ORIGIN'); - } - const originRecord = exact(receipt.origin, [ + if (!isRecord(originValue) || originValue.kind !== 'github') refuseCustody('BINDING_ORIGIN'); + const originRecord = exact(originValue, [ 'kind', 'repository', 'workflow', diff --git a/tools/cli-package/release-custody.mjs b/tools/cli-package/release-custody.mjs index 564f279e..a9223447 100644 --- a/tools/cli-package/release-custody.mjs +++ b/tools/cli-package/release-custody.mjs @@ -19,7 +19,7 @@ import { rmSync, writeSync, } from 'node:fs'; -import { join } from 'node:path'; +import { join, resolve } from 'node:path'; import { isDeepStrictEqual } from 'node:util'; import { ARCHIVE_LIMITS } from './archive.mjs'; import { readPrivateContainer } from './container.mjs'; @@ -78,7 +78,7 @@ export async function encryptCandidate(options) { const { scope, container, destination, privateRoot, now, run, gpg } = isRecord(options) ? options : {}; - admitted = admit(scope, container, destination); + admitted = admit(scope, container, destination, privateRoot); ports = { destination, privateRoot, now, run, gpg }; } catch (error) { // A host error's message names a path: only fixed custody codes leave. @@ -104,8 +104,9 @@ export async function encryptCandidate(options) { * Admit one encryption, reading the caller's scope and container once: the scope's rules, then a plain, * unsealed destination that holds exactly the candidate's tarball, all before any GnuPG call. * @param {unknown} scope @param {unknown} container @param {string} destination + * @param {string} privateRoot */ -function admit(scope, container, destination) { +function admit(scope, container, destination, privateRoot) { const { kind, recipient: given } = isRecord(scope) ? scope : {}; if ((kind !== 'production' && kind !== 'synthetic') || !isRecord(given)) refuseCustody('SCOPE'); if (kind === 'production' && !isProductionRecipient(given)) refuseCustody('RECIPIENT_SOURCE'); @@ -122,9 +123,11 @@ function admit(scope, container, destination) { if (!isSyntheticSource(manifest.source)) refuseCustody('FIXTURE_SCOPE'); recipient = { file: given.file, pins: given.pins }; } - if (present(join(destination, RELEASE_BINDING_FILE))) refuseCustody('ALREADY_SEALED'); - // A plain directory, as encryptOnce requires, resolved now so that nothing after the ciphertext is - // published can fail on it. + // Normalized absolute paths only: a trailing separator, `.` or `..` would make lstat follow a link. + if (resolve(privateRoot) !== privateRoot) refuseCustody('PRIVATE_ROOT'); + if (resolve(destination) !== destination) refuseCustody('PLACEMENT'); + // A plain directory, as encryptOnce requires, checked before anything is read through it and + // resolved now so that nothing after the ciphertext is published can fail on it. let resolved; try { if (!lstatSync(destination).isDirectory()) refuseCustody('PLACEMENT'); @@ -133,6 +136,7 @@ function admit(scope, container, destination) { if (error instanceof CustodyError) throw error; refuseCustody('PLACEMENT'); } + if (present(join(resolved, RELEASE_BINDING_FILE))) refuseCustody('ALREADY_SEALED'); // The sealer admits exactly the tarball and the ciphertext: a destination holding anything else, or a // missing or different tarball, would leave an unsealable ciphertext, so it refuses first. const tarballFile = tarballName(manifest.packageVersion); @@ -199,8 +203,9 @@ function publish(directory, bytes, write) { * @param {{origin?: unknown, write?: (path: string, bytes: Buffer) => void}} [options] * @returns {{receipt: ReturnType, bytes: Buffer}} */ -export function sealReleaseBinding(record, { origin, write = writePublicFile } = {}) { +export function sealReleaseBinding(record, options) { try { + const { origin, write = writePublicFile } = isRecord(options) ? options : {}; return seal(record, origin, write); } catch (error) { if (error instanceof CustodyError) throw error; diff --git a/tools/cli-package/release-custody.test.mjs b/tools/cli-package/release-custody.test.mjs index 064e9dca..8ef1254c 100644 --- a/tools/cli-package/release-custody.test.mjs +++ b/tools/cli-package/release-custody.test.mjs @@ -11,7 +11,7 @@ import { writeFileSync, } from 'node:fs'; import { createRequire, syncBuiltinESMExports } from 'node:module'; -import { join } from 'node:path'; +import { basename, join, relative } from 'node:path'; import { before, test } from 'node:test'; import { fresh, posixModes } from './candidate-fixture.mjs'; import { readPrivateContainer } from './container.mjs'; @@ -185,6 +185,57 @@ test('the gate refuses a wrong scope, recipient or candidate before any GnuPG ca ); assert.deepEqual(calls, [], code); } + // Only a normalized absolute path names the destination and the private root, so a trailing separator + // or `.` cannot make lstat follow a link; a link whose target holds a receipt refuses as a link. + if (posixModes) { + const target = fresh('linked-target'); + mkdirSync(target); + writeFileSync(join(target, tarballName(VERSION)), TARBALL); + const link = fresh('linked-destination'); + symlinkSync(target, link); + const sealedTarget = fresh('linked-sealed-target'); + mkdirSync(sealedTarget); + writeFileSync(join(sealedTarget, tarballName(VERSION)), TARBALL); + writeFileSync(join(sealedTarget, RELEASE_BINDING_FILE), '{}'); + const sealedLink = fresh('linked-sealed-destination'); + symlinkSync(sealedTarget, sealedLink); + for (const destination of [`${link}/`, `${link}/.`, sealedLink]) { + const { root } = places(); + const { run, calls } = recording(); + await assert.rejects( + encryptCandidate({ + scope: synthetic, + container: built.synthetic, + destination, + privateRoot: root, + run, + }), + refuses('PLACEMENT'), + destination, + ); + assert.deepEqual(calls, [], destination); + assert.deepEqual(readdirSync(target), [tarballName(VERSION)]); + } + } + { + const { root, destination } = places(); + for (const [code, options] of [ + ['PLACEMENT', { destination: relative(process.cwd(), destination), privateRoot: root }], + [ + 'PLACEMENT', + { destination: `${destination}/../${basename(destination)}`, privateRoot: root }, + ], + ['PRIVATE_ROOT', { destination, privateRoot: `${root}/` }], + ]) { + const { run, calls } = recording(); + await assert.rejects( + encryptCandidate({ scope: synthetic, container: built.synthetic, ...options, run }), + refuses(code), + JSON.stringify(options), + ); + assert.deepEqual(calls, []); + } + } // A tarball whose name differs only in case is not the tarball, even where the volume would open it. { const { root, destination } = places(); @@ -360,9 +411,19 @@ test('only a record the gate returned seals, and an existing receipt refuses bef refuses('ALREADY_SEALED'), ); assert.equal(readFileSync(join(destination, RELEASE_BINDING_FILE), 'utf8'), 'planted'); - // Calling without options refuses with a fixed code, never a host error. + // Calling without options, or with a throwing origin, refuses with a fixed code, never a host error. const again = await encrypted(); assert.throws(() => sealReleaseBinding(again.record), refuses('BINDING_ORIGIN')); + assert.throws(() => sealReleaseBinding(again.record, null), refuses('BINDING_ORIGIN')); + assert.throws( + () => + sealReleaseBinding(again.record, { + get origin() { + throw new Error(`at ${again.destination}`); + }, + }), + (error) => refuses('UNEXPECTED')(error) && !error.message.includes(again.destination), + ); noReceipt(again.destination); }); diff --git a/tools/cli-package/release-recipient.test.mjs b/tools/cli-package/release-recipient.test.mjs index d98df0da..a33a3622 100644 --- a/tools/cli-package/release-recipient.test.mjs +++ b/tools/cli-package/release-recipient.test.mjs @@ -67,6 +67,13 @@ test('the committed recipient directory is absent in Wave 3, so every production () => loadRecipientForTests({ root: join(fresh('missing'), 'release-recipient') }), refuses('PRODUCTION_RECIPIENT_ABSENT'), ); + // Only a missing root is absent: any other failure to reach it, such as a parent that is a file, is not. + const parentFile = fresh('parent-file'); + writeFileSync(parentFile, ''); + assert.throws( + () => loadRecipientForTests({ root: join(parentFile, 'release-recipient') }), + refuses('PRODUCTION_RECIPIENT_INVALID'), + ); }); test('a loaded recipient is frozen, branded, and its current pins come from an append-only history', () => { diff --git a/tools/cli-package/release-scope.test.mjs b/tools/cli-package/release-scope.test.mjs index 22ad2f4f..66e9e387 100644 --- a/tools/cli-package/release-scope.test.mjs +++ b/tools/cli-package/release-scope.test.mjs @@ -84,61 +84,61 @@ test('no product or tooling code reaches a test fixture, and only the loader rea }); /** - * Whether `text`, a file at repository-relative `path`, names repository-relative `target` by any quoted - * literal that resolves to it, as Node would resolve a relative specifier or a `new URL(…, - * import.meta.url)`: relative to the file, or to the repository root for a workflow (whose steps run - * there), a package manifest's scripts and a `tools/` path. Percent-escapes are decoded and a query or - * fragment is ignored. - * @param {string} path @param {string} text @param {string} target + * Whether `text` refers, by any quoted literal, to a file whose name is `name`, whatever directory it + * would resolve from: a package script runs in its package and a workflow step in its working directory, + * so no single base is right. Separators are normalized, a query or fragment is ignored, percent-escapes + * are decoded and case is ignored, as a case-insensitive volume would. Only one tracked file has the name. + * @param {string} text @param {string} name */ -function resolvesTo(path, text, target) { - const root = /(?:\.ya?ml|package\.json)$/u.test(path) ? '.' : posix.dirname(path); +function namesFile(text, name) { // Every run of non-space characters directly between two quotes; the closing quote may open the next // run, so a literal nested inside another (a workflow's `node -e "import('…')"`) is found too. for (const match of text.matchAll(/['"`]([^'"`\s]+)(?=['"`])/gu)) { - let specifier = match[1].replace(/[?#].*$/u, ''); + let specifier = match[1].replaceAll('\\', '/').replace(/[?#].*$/u, ''); try { specifier = decodeURIComponent(specifier); } catch { continue; } - if (posix.basename(specifier) !== posix.basename(target)) continue; - const base = specifier.startsWith('tools/') || specifier.startsWith('./tools/') ? '.' : root; - if (posix.normalize(posix.join(base, specifier)) === target) return true; + if (posix.basename(specifier).toLowerCase() === name) return true; } return false; } -test('the import match resolves every spelling, and only the real target', () => { - const target = 'tools/cli-package/custody.mjs'; - for (const [path, text] of [ - ['tools/cli-package/smoke/x.mjs', "import { encryptOnce } from '../custody.mjs';"], - ['tools/x.mjs', "import { encryptOnce } from './cli-package/custody.mjs';"], - ['tools/ci/x/y.mjs', 'const m = await import("../../cli-package/custody.mjs?v=1");'], - ['tools/cli-package/x.mjs', "import { encryptOnce } from './custody.mjs#fragment';"], - ['tools/cli-package/x.mjs', "import { encryptOnce } from './custody%2Emjs';"], - ['tools/cli-package/x.mjs', "const url = new URL('custody.mjs', import.meta.url);"], - ['.github/workflows/x.yml', 'run: node -e "import(\'./tools/cli-package/custody.mjs\')"'], - ['package.json', '"x": "node -e \\"import(\'./tools/cli-package/custody.mjs\')\\""'], - ['apps/cli/x.ts', "export * as custody from '../../tools/cli-package/custody.mjs';"], - ['tools/cli-package/x.mjs', 'require(`./custody.mjs`)'], +test('the reference match finds every spelling of the name, and only that name', () => { + for (const text of [ + "import { encryptOnce } from '../custody.mjs';", + "import { encryptOnce } from './cli-package/custody.mjs';", + 'const m = await import("../../cli-package/custody.mjs?v=1");', + "import { encryptOnce } from './custody.mjs#fragment';", + "import { encryptOnce } from './custody%2Emjs';", + "const url = new URL('custody.mjs', import.meta.url);", + "const url = new URL('.\\\\custody.mjs', import.meta.url);", + "const url = new URL('Custody.mjs', import.meta.url);", + 'run: node -e "import(\'./tools/cli-package/custody.mjs\')"', + '"x": "node -e \\"import(\'../../tools/cli-package/custody.mjs\')\\""', + "export * as custody from '../../tools/cli-package/custody.mjs';", + 'require(`./custody.mjs`)', ]) { - assert.equal(resolvesTo(path, text, target), true, `${path}: ${text}`); + assert.equal(namesFile(text, 'custody.mjs'), true, text); } - for (const [path, text] of [ - ['tools/cli-package/x.mjs', "import { refuseCustody } from './custody-error.mjs';"], - ['tools/cli-package/x.mjs', "import { encryptCandidate } from './release-custody.mjs';"], - ['tools/x.mjs', "import { x } from './custody.mjs';"], - ['tools/cli-package/x.mjs', '// encryptOnce lives in custody.mjs'], - ['tools/cli-package/x.mjs', "import { x } from './custody.mjs.bak';"], + for (const text of [ + "import { refuseCustody } from './custody-error.mjs';", + "import { encryptCandidate } from './release-custody.mjs';", + '// encryptOnce lives in custody.mjs', + "import { x } from './custody.mjs.bak';", + "the container's bound (container.mjs, custody.mjs)", ]) { - assert.equal(resolvesTo(path, text, target), false, `${path}: ${text}`); + assert.equal(namesFile(text, 'custody.mjs'), false, text); } }); -test('only the release gate imports encryptOnce, by any spelling', () => { - const target = 'tools/cli-package/custody.mjs'; - assert.deepEqual(code.filter((path) => resolvesTo(path, read(path), target)).sort(), [ +test('only the release gate refers to custody.mjs, by any spelling', () => { + assert.equal( + tracked.filter((path) => posix.basename(path).toLowerCase() === 'custody.mjs').length, + 1, + ); + assert.deepEqual(code.filter((path) => namesFile(read(path), 'custody.mjs')).sort(), [ 'tools/cli-package/release-custody.mjs', ]); }); @@ -159,6 +159,38 @@ test('no product path or workflow names a release module yet', () => { ]); }); +/** + * Whether every dynamic import in `source` takes exactly one string literal, so that a closure scan + * reads every module the source can load. + * @param {string} source + */ +function dynamicImportsAreLiteral(source) { + for (const match of source.matchAll(/\bimport\s*\(/gu)) { + const rest = source.slice(match.index + match[0].length); + if (!/^\s*(['"])[^'"\n]+\1\s*\)/u.test(rest)) return false; + } + return true; +} + +test('a dynamic import is admitted only as one string literal', () => { + for (const source of [ + "await import('./archive.mjs');", + 'import("node:fs")', + "import ( './x.mjs' )", + ]) { + assert.equal(dynamicImportsAreLiteral(source), true, source); + } + for (const source of [ + "import('node:child' + '_process')", + "import('./' + name)", + 'import(`./x.mjs`)', + 'import(name)', + "import('./x.mjs', { with: {} })", + ]) { + assert.equal(dynamicImportsAreLiteral(source), false, source); + } +}); + /** Every specifier a module's import closure reaches, relative files followed. @param {string} entry */ function closure(entry) { const seen = new Set(); @@ -211,9 +243,10 @@ test("the binding receipt's closure starts no process, so reuse makes no GnuPG c const source = readFileSync(join(here, file), 'utf8'); assert.doesNotMatch( source, - /\b(?:getBuiltinModule|createRequire|process\.binding|process\.dlopen|process\.execve)\b|\brequire\s*\(|\bimport\(\s*[^'"\s]/u, + /\b(?:getBuiltinModule|createRequire|process\.binding|process\.dlopen|process\.execve)\b|\brequire\s*\(/u, file, ); + assert.equal(dynamicImportsAreLiteral(source), true, file); } // The scan reads the imports it claims to: the gate's closure reaches GnuPG. const gate = closure('release-custody.mjs'); From 563402e621504e37cba5bc41e2ccfbc1c3f1dc9e Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 26 Oct 2026 06:39:18 +0300 Subject: [PATCH 86/97] fix(repo): address the third Step 9b.2 review round Round 3 at 463a047c: the plan and integration lenses were clean and the security sitting reported lows only. The tests lens reported four mediums, all in tests or static controls, the code itself correct: - a non-EEXIST link failure was untested; an injected EPERM now refuses UNEXPECTED, publishes nothing and keeps the record sealable; - the gate's refusal of a linked tarball was untested; - no test pinned the patterns' trailing anchors; - a template literal hid a custody.mjs reference from the control; the control now refuses any mention of the name outside the gate, comments included, after JavaScript and percent escapes, separators and case are normalized. Lows corrected: the closure scan reads every specifier after from or import whatever the spacing; new tests for single reads, throwing options, same-count renames, the toolchain bounds and the receipt bytes reuse returns; the contract's diagnostics names every path's code. The generated sweep gained throw, anchor, logical-operator, bound and statement-deletion operators: 233 mutants, three more gaps fixed, ten argued equivalent. Refs: #46, ADR-0111 Co-Authored-By: Claude Opus 5.5 --- docs/reference/cli/packaging.md | 20 +++-- .../phase-2.5.5-hardening-and-remediation.md | 40 +++++++++ tools/cli-package/release-binding.mjs | 2 +- tools/cli-package/release-binding.test.mjs | 83 ++++++++++++++++- tools/cli-package/release-custody.test.mjs | 77 ++++++++++++++++ tools/cli-package/release-scope.test.mjs | 90 ++++++++++++++----- 6 files changed, 277 insertions(+), 35 deletions(-) diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 700c88b9..8499700a 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -713,21 +713,23 @@ refuses every artifact. Its review and acceptance are recorded in the renames and removes nothing. Its module imports nothing that can start a process, so reuse makes no GnuPG call, and it cannot re-check the ciphertext's recipients: a named limit. - **Diagnostics.** Every refusal is a fixed `PACKED_CLI_CUSTODY_`. A host failure is `UNEXPECTED`, - except while loading the recipient (`PRODUCTION_RECIPIENT_INVALID`) or reading a named file, which - carries that file's code (`CANDIDATE_ASSET`, `CIPHERTEXT_DRIFT`, `TARBALL_DRIFT`). No path, key or - receipt content surfaces. + except on a named path, which carries that path's code: `PLACEMENT` for the destination, + `CANDIDATE_ASSET`, `CIPHERTEXT_DRIFT` and `TARBALL_DRIFT` for the files read, and + `PRODUCTION_RECIPIENT_INVALID` while loading the recipient; inside encryption, `encryptOnce`'s own + codes apply. No path, key or receipt content surfaces. - **Where it runs.** `release-recipient.test.mjs`, `release-binding.test.mjs`, `release-custody.test.mjs` and `release-scope.test.mjs` join `test:acceptance-tools`, as the 9b.1 tests do; the custody test needs GnuPG ≥ 2.2.8. `release-scope.test.mjs` pins the separation statically: - no product or tooling code reaches a test fixture, and only the loader reads the fixture pins; - - only the release gate refers to `custody.mjs`, the one tracked file of that name, by any quoted - literal in source, workflow and package-script text, whatever directory it would resolve from: - separators, queries, fragments, percent-escapes and case are normalized; + - only the release gate mentions `custody.mjs`, the one tracked file of that name, anywhere in source, + workflow and package-script text, comments included, once JavaScript and percent escapes, + separators and case are normalized; a path built at run time from pieces is beyond any static + control; - the loader is never called with an argument, and its test seam appears in no product or tooling file (permanent controls); - - the receipt module's import closure holds only its own files and listed Node builtins, with no - package, process module or `require`-like call, and every dynamic import takes exactly one string - literal; + - the receipt module's import closure, read from every `from` or `import` specifier whatever the + spacing, holds only its own files and listed Node builtins, with no package, process module or + `require`-like call, and every dynamic import takes exactly one string literal; - the interim guard still refuses a complete, valid synthetic candidate; - until Step 9d no product path or workflow names a release module, and until Wave 4b no recipient directory is committed. diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index a7d9c733..9a8ebef5 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -10540,3 +10540,43 @@ Nothing outside the gate and the tests calls it, and the gate requires normalize **Sweeps at the round 2 corrections.** The generated sweep ran 135 mutants and killed 131; the four survivors are the equivalents named above. Of 40 named mutants, 38 are killed; the two survivors are the equivalents named above. + +**Author evidence at `463a047c`.** The full `pnpm run ci` passed plain and alone (exit 0): `test:acceptance-tools` +1,023 of 1,023, all 25 product map controls, and every causal-control family. On Ubuntu 24.04 with GnuPG 2.4.4, the +89 tests ran: 88 passed and the case-insensitive-volume case skipped. + +**Round 3 at `463a047c` — NONCLEAN.** Five lenses and a critic ran, every finding adversarially verified. The plan and +integration lenses were clean, and the security sitting reported lows only. The tests lens reported four mediums, all +in tests or static controls; the code they cover was correct: +- **A non-`EEXIST` link failure was untested.** Turning that `throw` into a `return` passed every test, so a seal could + have reported success with no receipt. An injected `EPERM` link now refuses `UNEXPECTED` with no host text, publishes + nothing and leaves the record sealable (T4-1). +- **The gate's refusal of a linked tarball was untested.** A tarball that is a link to a file with the right bytes + now refuses `TARBALL_DRIFT` before any GnuPG call (T4-2). +- **No test pinned the patterns' trailing anchors.** A suffix after each valid value now refuses (T4-3). +- **A template literal escaped the `custody.mjs` control.** A substitution containing a space hid + `` `${join(here, base)}/custody.mjs` ``, and whitespace-bearing scripts, unquoted arguments and JavaScript escapes + hid other spellings. The control no longer looks for quoted specifiers: after JavaScript and percent escapes, + separators and case are normalized, it refuses any mention of the name outside the gate, comments included + (T4-4, PLAN-9B2-R3-1, SIT3-L2). + +Lows corrected: +- the closure scan reads every specifier after `from` or `import`, whatever the spacing, so spaced, compact and + multi-line imports are seen (SIT3-L1, T4-5); +- new tests cover single reads of the gate's recipient, the validator's origin and inputs, throwing options on the gate + and on reuse, same-count renames for the sealer and reuse, the toolchain's length bounds, and the receipt bytes + returned by reuse while the file changes (COR3-R3-1, COR3-R3-2, T4-6, T4-7, T4-8); +- the contract's diagnostics bullet names every path's code (CRIT-9B2-R3-1). + +**Sweeps at the round 3 corrections.** The generated sweep gained the operator classes the rounds had found by hand: a +`throw` turned into a `return`, each regex anchor removed, each `||` and `&&` swapped inside one-line conditions, each +integer bound raised by one, and each single-line call statement deleted. It ran 233 mutants. Three survivors were +real gaps, the `typescript`, `tsup` and `esbuild` length bounds, and gained cases. Ten are equivalent: +- the four named above; +- `isRecord`'s two operator swaps and the history guard's `||`, which let through only values that still refuse with + the same code; +- `string()`'s default maximum, used only by fixed-length patterns; +- deleting `writePublicFile`'s `fsync` or `close`, which guard durability and descriptor hygiene, not anything a test + observes. + +Of 40 named mutants, 38 are killed; the two survivors are the equivalents named above. diff --git a/tools/cli-package/release-binding.mjs b/tools/cli-package/release-binding.mjs index dd7ed41a..712384c8 100644 --- a/tools/cli-package/release-binding.mjs +++ b/tools/cli-package/release-binding.mjs @@ -19,7 +19,7 @@ import { isProductionRecipient, isSyntheticSource } from './release-recipient.mj /** The receipt's file name beside the ciphertext, as the interim release guard keys on it. */ export const RELEASE_BINDING_FILE = 'release-binding-receipt.json'; export const RELEASE_BINDING_KIND = 'release_binding'; -/** The custody container's 128 MiB total plus the 64 KiB a ciphertext may add (container.mjs, custody.mjs). */ +/** The custody container's 128 MiB total plus the 64 KiB a ciphertext may add to it (9b.1's bounds). */ export const CIPHERTEXT_BYTES = 128 * 1024 * 1024 + 64 * 1024; const RECEIPT_BYTES = 64 * 1024; const MAX_INPUTS = 64; diff --git a/tools/cli-package/release-binding.test.mjs b/tools/cli-package/release-binding.test.mjs index b85475a8..904f8559 100644 --- a/tools/cli-package/release-binding.test.mjs +++ b/tools/cli-package/release-binding.test.mjs @@ -8,6 +8,7 @@ import { readFileSync, readdirSync, readlinkSync, + renameSync, rmSync, symlinkSync, writeFileSync, @@ -278,6 +279,26 @@ test('every value has its exact format', () => { (copy) => (copy.origin.runId = `x${copy.origin.runId}`), (copy) => (copy.origin.runAttempt = `x${copy.origin.runAttempt}`), (copy) => (copy.inputs = { 'package.json': `x${'a'.repeat(64)}` }), + // ...and at the end: a suffix after a valid value refuses. + (copy) => (copy.source.commit = `${copy.source.commit}x`), + (copy) => (copy.source.tree = `${copy.source.tree}x`), + (copy) => (copy.tarball.sha256 = `${copy.tarball.sha256}a`), + (copy) => (copy.recipient.subkey = `${copy.recipient.subkey}A`), + (copy) => (copy.toolchain.node = `${copy.toolchain.node}x`), + (copy) => (copy.toolchain.typescript = `${copy.toolchain.typescript} `), + (copy) => (copy.toolchain.gnupg = `${copy.toolchain.gnupg}x`), + (copy) => (copy.target = `${copy.target}x`), + (copy) => (copy.origin.repository = `${copy.origin.repository} x`), + (copy) => (copy.origin.workflow = `${copy.origin.workflow}\u0007`), + (copy) => (copy.origin.runId = `${copy.origin.runId}x`), + (copy) => (copy.origin.runAttempt = `${copy.origin.runAttempt}x`), + (copy) => (copy.inputs = { 'package.json': `${'a'.repeat(64)}a` }), + // The toolchain's length bounds: one past each refuses. + (copy) => (copy.toolchain.node = `v1.1.${'1'.repeat(36)}`), + ...['pnpm', 'typescript', 'tsup', 'esbuild'].map( + (tool) => (copy) => (copy.toolchain[tool] = `1.1.${'1'.repeat(97)}`), + ), + (copy) => (copy.toolchain.gnupg = `1.1.${'1'.repeat(37)}`), // A value whose string form matches is still not a string. (copy) => (copy.toolchain.node = [copy.toolchain.node]), (copy) => (copy.source.commit = [copy.source.commit]), @@ -316,6 +337,15 @@ test('every value has its exact format', () => { copy.inputs = { '01': 'a'.repeat(64), 'package.json': 'b'.repeat(64) }; }); assert.deepEqual(validateReleaseBinding(origins, synthetic).origin, origins.origin); + // The toolchain's length bounds are inclusive. + const tools = changed(value, (copy) => { + copy.toolchain.node = `v1.1.${'1'.repeat(35)}`; + for (const tool of ['pnpm', 'typescript', 'tsup', 'esbuild']) { + copy.toolchain[tool] = `1.1.${'1'.repeat(96)}`; + } + copy.toolchain.gnupg = `1.1.${'1'.repeat(36)}`; + }); + assert.deepEqual(validateReleaseBinding(tools, synthetic).toolchain, tools.toolchain); assert.deepEqual(validateReleaseBinding(single, synthetic).inputs, { 'package.json': 'a'.repeat(64), }); @@ -636,6 +666,15 @@ test('a retained candidate whose assets or entries differ from its receipt refus ['CANDIDATE_ENTRIES', (directory) => writeFileSync(join(directory, 'notes.txt'), 'extra')], ['CANDIDATE_ENTRIES', (directory) => rmSync(join(directory, ciphertextFileName(VERSION)))], ['CANDIDATE_ENTRIES', (directory) => rmSync(join(directory, RELEASE_BINDING_FILE))], + // Same count, another name: the listing is compared by name, not by size. + [ + 'CANDIDATE_ENTRIES', + (directory) => + renameSync( + join(directory, tarballName(VERSION)), + join(directory, tarballName(VERSION).toUpperCase()), + ), + ], [ 'CANDIDATE_ASSET', (directory) => { @@ -795,6 +834,21 @@ test('each receipt field is read once, and a throwing accessor refuses with a fi }); assert.equal(validateReleaseBinding(clean, productionExpected()).source.dirty, false); assert.equal(dirtyReads, 1); + // The origin and the inputs are read once each. + const counts = { origin: 0, inputs: 0 }; + const counted = receipt(); + for (const key of ['origin', 'inputs']) { + const original = counted[key]; + Object.defineProperty(counted, key, { + enumerable: true, + get() { + counts[key] += 1; + return original; + }, + }); + } + validateReleaseBinding(counted, synthetic); + assert.deepEqual(counts, { origin: 1, inputs: 1 }); const throwing = receipt(); Object.defineProperty(throwing, 'target', { enumerable: true, @@ -850,10 +904,37 @@ test('reuse reads its expectation once, and a vanished asset refuses as that ass fs.lstatSync = lstat; syncBuiltinESMExports(); } - // A malformed call refuses with a fixed code. + // A malformed call refuses with a fixed code, and a throwing option with no host text. for (const options of [undefined, null, 'options']) { assert.throws(() => verifyRetainedCandidate(options), refuses('BINDING_SCOPE')); } + assert.throws( + () => + verifyRetainedCandidate({ + get directory() { + throw new Error(`at ${scratch}`); + }, + }), + refuses('UNEXPECTED'), + ); + // The receipt bytes returned are the ones verified, even when the file changes during verification. + const changing = candidate(); + const receiptPath = join(changing, RELEASE_BINDING_FILE); + const original = readFileSync(receiptPath); + const tarballPath = join(changing, tarballName(VERSION)); + fs.lstatSync = (target, ...rest) => { + if (target === tarballPath) writeFileSync(receiptPath, Buffer.from('{"replaced": true}\n')); + return lstat(target, ...rest); + }; + syncBuiltinESMExports(); + let verified; + try { + verified = verifyRetainedCandidate({ directory: changing, expected: synthetic }); + } finally { + fs.lstatSync = lstat; + syncBuiltinESMExports(); + } + assert.deepEqual(verified.receiptBytes, original); }); test('reuse needs a valid expectation, and a host failure refuses with a fixed code and no path', () => { diff --git a/tools/cli-package/release-custody.test.mjs b/tools/cli-package/release-custody.test.mjs index 8ef1254c..29848508 100644 --- a/tools/cli-package/release-custody.test.mjs +++ b/tools/cli-package/release-custody.test.mjs @@ -6,6 +6,7 @@ import { mkdirSync, readFileSync, readdirSync, + renameSync, rmSync, symlinkSync, writeFileSync, @@ -168,6 +169,17 @@ test('the gate refuses a wrong scope, recipient or candidate before any GnuPG ca mkdirSync(join(destination, tarballName(VERSION))); }, ], + // A link to a file with the right bytes is not an ordinary tarball either. + [ + 'TARBALL_DRIFT', + (destination) => { + if (!posixModes) return rmSync(join(destination, tarballName(VERSION))); + const target = fresh('linked-tarball-target'); + writeFileSync(target, TARBALL); + rmSync(join(destination, tarballName(VERSION))); + symlinkSync(target, join(destination, tarballName(VERSION))); + }, + ], ]) { const { root, destination } = places(); change(destination); @@ -258,6 +270,39 @@ test('the gate refuses a wrong scope, recipient or candidate before any GnuPG ca for (const options of [undefined, null, 'options']) { await assert.rejects(encryptCandidate(options), refuses('SCOPE')); } + await assert.rejects( + encryptCandidate({ + get scope() { + throw new Error('/private/host/path'); + }, + }), + (error) => refuses('UNEXPECTED')(error) && !error.message.includes('/private/host/path'), + ); + // The recipient is read once too: the brand check sees the value encrypted to. + { + let reads = 0; + const counted = { + kind: 'production', + get recipient() { + reads += 1; + return { file: ephemeral.file, current: ephemeral.pins, history: [] }; + }, + }; + const { root, destination } = places(); + const { run, calls } = recording(); + await assert.rejects( + encryptCandidate({ + scope: counted, + container: built.clean, + destination, + privateRoot: root, + run, + }), + refuses('RECIPIENT_SOURCE'), + ); + assert.deepEqual(calls, []); + assert.equal(reads, 1); + } // The scope is read once: a kind that changes after its first read cannot skip the brand check. { let reads = 0; @@ -437,6 +482,15 @@ test('the sealer refuses a candidate directory that is not exactly its tarball a ['CANDIDATE_ENTRIES', (destination) => writeFileSync(join(destination, 'notes.txt'), 'x')], ['CANDIDATE_ENTRIES', (destination) => rmSync(join(destination, tarballName(VERSION)))], ['CANDIDATE_ENTRIES', (destination, record) => rmSync(join(destination, record.name))], + // Same count, another name: the listing is compared by name, not by size. + [ + 'CANDIDATE_ENTRIES', + (destination) => + renameSync( + join(destination, tarballName(VERSION)), + join(destination, tarballName(VERSION).toUpperCase()), + ), + ], [ 'CIPHERTEXT_DRIFT', (destination, record) => appendFileSync(join(destination, record.name), 'x'), @@ -539,6 +593,29 @@ test('the receipt is published once and atomically, whatever happens to the writ // The record survives a failed seal, so the candidate can still be sealed. sealReleaseBinding(failed.record, { origin: origin(SYNTHETIC_SOURCE.commit) }); assert.ok(readdirSync(failed.destination).includes(RELEASE_BINDING_FILE)); + // A link that fails for another reason publishes nothing, reports no host text and keeps the record. + const unlinked = await encrypted(); + const fs = createRequire(import.meta.url)('node:fs'); + const link = fs.linkSync; + fs.linkSync = (existing) => { + throw Object.assign(new Error(`EPERM: ${existing}`), { code: 'EPERM' }); + }; + syncBuiltinESMExports(); + try { + assert.throws( + () => sealReleaseBinding(unlinked.record, { origin: origin(SYNTHETIC_SOURCE.commit) }), + (error) => refuses('UNEXPECTED')(error) && !error.message.includes(unlinked.destination), + ); + } finally { + fs.linkSync = link; + syncBuiltinESMExports(); + } + assert.deepEqual( + readdirSync(unlinked.destination).sort(), + [ciphertextFileName(VERSION), tarballName(VERSION)].sort(), + ); + sealReleaseBinding(unlinked.record, { origin: origin(SYNTHETIC_SOURCE.commit) }); + assert.ok(readdirSync(unlinked.destination).includes(RELEASE_BINDING_FILE)); }); test( diff --git a/tools/cli-package/release-scope.test.mjs b/tools/cli-package/release-scope.test.mjs index 66e9e387..6c37ac81 100644 --- a/tools/cli-package/release-scope.test.mjs +++ b/tools/cli-package/release-scope.test.mjs @@ -84,39 +84,44 @@ test('no product or tooling code reaches a test fixture, and only the loader rea }); /** - * Whether `text` refers, by any quoted literal, to a file whose name is `name`, whatever directory it - * would resolve from: a package script runs in its package and a workflow step in its working directory, - * so no single base is right. Separators are normalized, a query or fragment is ignored, percent-escapes - * are decoded and case is ignored, as a case-insensitive volume would. Only one tracked file has the name. + * Whether `text` mentions a file named `name` anywhere, quoted or not, comments included: a package script + * runs in its package and a workflow step in its working directory, and a template literal can build the + * rest of the path, so no resolution is attempted. JavaScript and percent escapes are decoded, separators + * normalized and case ignored first. Only one tracked file has the name, so any mention is a reference. * @param {string} text @param {string} name */ function namesFile(text, name) { - // Every run of non-space characters directly between two quotes; the closing quote may open the next - // run, so a literal nested inside another (a workflow's `node -e "import('…')"`) is found too. - for (const match of text.matchAll(/['"`]([^'"`\s]+)(?=['"`])/gu)) { - let specifier = match[1].replaceAll('\\', '/').replace(/[?#].*$/u, ''); - try { - specifier = decodeURIComponent(specifier); - } catch { - continue; - } - if (posix.basename(specifier).toLowerCase() === name) return true; - } - return false; + const decoded = text + .replace(/\\\r?\n/gu, '') + .replace(/\\x([0-9a-f]{2})|\\u\{([0-9a-f]+)\}|\\u([0-9a-f]{4})/giu, (_, x, braced, u) => + String.fromCodePoint(Number.parseInt(x ?? braced ?? u, 16)), + ) + .replace(/%([0-9a-f]{2})/giu, (_, hex) => String.fromCharCode(Number.parseInt(hex, 16))) + .replaceAll('\\', '/') + .toLowerCase(); + const escaped = name.toLowerCase().replaceAll('.', '\\.'); + return new RegExp(`(? { for (const text of [ "import { encryptOnce } from '../custody.mjs';", - "import { encryptOnce } from './cli-package/custody.mjs';", 'const m = await import("../../cli-package/custody.mjs?v=1");', "import { encryptOnce } from './custody.mjs#fragment';", "import { encryptOnce } from './custody%2Emjs';", + "import { encryptOnce } from './%63ustody.mjs';", + "import { encryptOnce } from './custody\\u002emjs';", + "import { encryptOnce } from './custody\\x2emjs';", + "import { encryptOnce } from './cus\\u{74}ody.mjs';", "const url = new URL('custody.mjs', import.meta.url);", "const url = new URL('.\\\\custody.mjs', import.meta.url);", "const url = new URL('Custody.mjs', import.meta.url);", + 'const m = (here, base) => import(`${join(here, base)}/custody.mjs`);', 'run: node -e "import(\'./tools/cli-package/custody.mjs\')"', - '"x": "node -e \\"import(\'../../tools/cli-package/custody.mjs\')\\""', + '"a": "node --import ../../tools/cli-package/custody.mjs run.mjs"', + '"b": "node -e \\"import(process.argv[1])\\" ../tools/cli-package/custody.mjs"', + 'run: node tools/cli-package/custody.mjs', + '// encryptOnce lives in custody.mjs', "export * as custody from '../../tools/cli-package/custody.mjs';", 'require(`./custody.mjs`)', ]) { @@ -125,9 +130,9 @@ test('the reference match finds every spelling of the name, and only that name', for (const text of [ "import { refuseCustody } from './custody-error.mjs';", "import { encryptCandidate } from './release-custody.mjs';", - '// encryptOnce lives in custody.mjs', "import { x } from './custody.mjs.bak';", - "the container's bound (container.mjs, custody.mjs)", + "import { x } from './my-custody.mjs';", + "import { x } from './custodyxmjs';", ]) { assert.equal(namesFile(text, 'custody.mjs'), false, text); } @@ -191,6 +196,46 @@ test('a dynamic import is admitted only as one string literal', () => { } }); +/** + * Every module specifier `source` names: each string literal right after `from` or `import`, with or + * without a parenthesis, whatever the spacing, so a compact, spaced or multi-line import is read too. + * @param {string} source + */ +function specifiersOf(source) { + // A keyword, never a member call such as `Buffer.from('…')`. + return [...source.matchAll(/(? match[2], + ); +} + +test('the specifier scan reads every spelling of an import', () => { + assert.deepEqual( + specifiersOf( + [ + "import { a } from './a.mjs';", + "import{b}from'./b.mjs';", + "import 'node:fs';", + "import {\n c, // the container's\n} from './c.mjs';", + "export * as d from './d.mjs';", + "export { e } from './e.mjs';", + "const f = await import ( 'node:child_process' );", + 'const g = import("./g.mjs");', + "const magic = Buffer.from('ustar\\0');", + ].join('\n'), + ), + [ + './a.mjs', + './b.mjs', + 'node:fs', + './c.mjs', + './d.mjs', + './e.mjs', + 'node:child_process', + './g.mjs', + ], + ); +}); + /** Every specifier a module's import closure reaches, relative files followed. @param {string} entry */ function closure(entry) { const seen = new Set(); @@ -199,10 +244,7 @@ function closure(entry) { if (seen.has(path)) return; seen.add(path); const source = readFileSync(path, 'utf8'); - for (const match of source.matchAll( - /^\s*import\s+(?:[^'";]*?\s+from\s+)?['"]([^'"]+)['"]|^\s*export\s+(?:\*(?:\s+as\s+\w+)?|\{[^}]*\})\s+from\s+['"]([^'"]+)['"]|\bimport\(\s*['"]([^'"]+)['"]\s*\)/gmu, - )) { - const specifier = match[1] ?? match[2] ?? match[3]; + for (const specifier of specifiersOf(source)) { specifiers.add(specifier); if (specifier.startsWith('.')) visit(join(dirname(path), specifier)); } From 5a328bca302dcda1b69f826da46d45955bde02c0 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 26 Oct 2026 08:51:02 +0300 Subject: [PATCH 87/97] docs(repo): record the Step 9b.2 acceptance Round 4 at 563402e6, a new complete cumulative review of 819f8cbb..563402e6 with every reviewer in its own worktree, reported no blocker, high or medium finding and no regression; its separate crypto/custody security sitting found lows only. The full pnpm run ci passed plain and alone at that head (test:acceptance-tools 1024 of 1024, all 25 product map controls, every causal-control family), and the custody and release tests passed on Ubuntu 24.04 with GnuPG 2.4.4. Two lows are corrected here (the contract's diagnostics and the custody.mjs control's escape claim); the rest are recorded in deferred-tasks.md with their re-checks. This commit changes no code. Step 9c is next. Refs: #46, ADR-0111 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 +- CLAUDE.md | 4 +- docs/reference/cli/README.md | 2 +- docs/reference/cli/packaging.md | 16 ++++---- docs/roadmap/current.md | 6 +-- docs/roadmap/deferred-tasks.md | 38 ++++++++++++++++++- .../phase-2.5.5-hardening-and-remediation.md | 37 +++++++++++++++++- 7 files changed, 89 insertions(+), 18 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b2af541d..a899dc5a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,7 +117,7 @@ explicit; source CI does not certify later documentation commits. The joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now -accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI and Step 9b.1 are accepted too, Step 9b.2 is implemented and under review, and Steps 9c–19 have not started) and whole-wave acceptance remains Step 19. +accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI and Steps 9b.1 and 9b.2 are accepted too, and Steps 9c–19 have not started) and whole-wave acceptance remains Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification @@ -130,7 +130,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25): round 1 reported no blocker, high or medium finding and no regression, with a clean security sitting; every product file and directory identity check now compares exact bigints, its low findings are recorded in deferred-tasks.md, and its Windows clause closes after three green advisory Windows runs ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)). Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b-preflight-decisions--2026-10-25)); Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is independently accepted on `development` at `147c2788` (2026-10-25): rounds 1–3 were not accepted (one, one and two mediums, each corrected), and round 4 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-acceptance--2026-10-25)). Step 9b.2 (the release binding receipt v1, verified ciphertext reuse, the production recipient loader, absent in Wave 3, and scope separation; synthetic recipients only) is implemented on `development` and under review ([decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b2-preflight-decisions--2026-10-25), [checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b2-implementation-checkpoint--2026-10-25)). `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25): round 1 reported no blocker, high or medium finding and no regression, with a clean security sitting; every product file and directory identity check now compares exact bigints, its low findings are recorded in deferred-tasks.md, and its Windows clause closes after three green advisory Windows runs ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)). Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b-preflight-decisions--2026-10-25)); Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is independently accepted on `development` at `147c2788` (2026-10-25): rounds 1–3 were not accepted (one, one and two mediums, each corrected), and round 4 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-acceptance--2026-10-25)). Step 9b.2 (the release binding receipt v1, verified ciphertext reuse, the production recipient loader, absent in Wave 3, and scope separation; synthetic recipients only) is independently accepted on `development` at `563402e6` (2026-10-26): an internal rigor pass and rounds 1–3 were not accepted (mediums each time, all corrected), and round 4 reported no blocker, high or medium finding and no regression, its security sitting lows only; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b2-acceptance--2026-10-26)). Step 9c is next. `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. See [docs/roadmap/current.md](docs/roadmap/current.md) for live status. diff --git a/CLAUDE.md b/CLAUDE.md index 94669cb1..68cdba2d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -206,7 +206,7 @@ explicit; source CI does not certify later documentation commits. The joins exact evidence and historical corrections. Steps 1–8 are accepted; **Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now -accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI and Step 9b.1 are accepted too, Step 9b.2 is implemented and under review, and Steps 9c–19 have not started) and whole-wave acceptance remains Step 19. +accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI and Steps 9b.1 and 9b.2 are accepted too, and Steps 9c–19 have not started) and whole-wave acceptance remains Step 19. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) records what merged and its review attributions. The final head `a3e164b5` (corrections from a further independent review, including a secret-scrubbing rule) merged with author verification @@ -219,7 +219,7 @@ give every open PR #95 finding an owner and record 29 increments (no new ADR). T maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25): round 1 reported no blocker, high or medium finding and no regression, with a clean security sitting; every product file and directory identity check now compares exact bigints, its low findings are recorded in deferred-tasks.md, and its Windows clause closes after three green advisory Windows runs ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)). Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b-preflight-decisions--2026-10-25)); Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is independently accepted on `development` at `147c2788` (2026-10-25): rounds 1–3 were not accepted (one, one and two mediums, each corrected), and round 4 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-acceptance--2026-10-25)). Step 9b.2 (the release binding receipt v1, verified ciphertext reuse, the production recipient loader, absent in Wave 3, and scope separation; synthetic recipients only) is implemented on `development` and under review ([decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b2-preflight-decisions--2026-10-25), [checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b2-implementation-checkpoint--2026-10-25)). `development` is pushed after each accepted increment. No provider, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25): round 1 reported no blocker, high or medium finding and no regression, with a clean security sitting; every product file and directory identity check now compares exact bigints, its low findings are recorded in deferred-tasks.md, and its Windows clause closes after three green advisory Windows runs ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)). Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b-preflight-decisions--2026-10-25)); Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is independently accepted on `development` at `147c2788` (2026-10-25): rounds 1–3 were not accepted (one, one and two mediums, each corrected), and round 4 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-acceptance--2026-10-25)). Step 9b.2 (the release binding receipt v1, verified ciphertext reuse, the production recipient loader, absent in Wave 3, and scope separation; synthetic recipients only) is independently accepted on `development` at `563402e6` (2026-10-26): an internal rigor pass and rounds 1–3 were not accepted (mediums each time, all corrected), and round 4 reported no blocker, high or medium finding and no regression, its security sitting lows only; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b2-acceptance--2026-10-26)). Step 9c is next. `development` is pushed after each accepted increment. No provider, credential, billing or new ADR approval is pending for the approved implementation plan. For live status, per-PR history, milestone dates, and open obligations, see the canonical diff --git a/docs/reference/cli/README.md b/docs/reference/cli/README.md index 0bc7fd30..f8c11ca3 100644 --- a/docs/reference/cli/README.md +++ b/docs/reference/cli/README.md @@ -13,5 +13,5 @@ Part of [reference/](../README.md). | [commands.md](commands.md) | The full `relavium` command surface (run, chat, chat-resume/list/export, agent run, list, create, import, export, logs, gate, init, provider) plus interactive TUI vs CI JSON mode. | | [home.md](home.md) | The bare-invocation interactive **Home** — the TTY gate, the read-only management strip over `history.db`, the Home↔chat mode machine + keymap, bracketed paste, the min-terminal degrade, and the signal/exit-code lifecycle (2.5.B / ADR-0054). | | [chat-session.md](chat-session.md) | The `relavium chat` agent-session REPL — entry, agent/model selection, the multi-turn loop, streaming, tool availability, the `--json` session-event stream, and exit code 4. | -| [packaging.md](packaging.md) | Wave 3 finite runtime graph, installed-package consumers and Step 8 native activation accepted on `development`; the Step 9a.1 candidate build and finalizer are accepted on `development` and Step 9a.2a's private manifest and symbolication are accepted on `development` and Step 9a.2b's product map controls and `package_test` receipt are accepted on `development`; Step 9b.1's private container, isolated GnuPG runner and encrypt-once library are accepted on `development` (synthetic recipients only); Step 9b.2's ciphertext reuse, release binding receipt v1, absent production loader and scope separation are implemented on `development` and under review; the remaining Step 9 custody and release gates remain staged. | +| [packaging.md](packaging.md) | Wave 3 finite runtime graph, installed-package consumers and Step 8 native activation accepted on `development`; the Step 9a.1 candidate build and finalizer are accepted on `development` and Step 9a.2a's private manifest and symbolication are accepted on `development` and Step 9a.2b's product map controls and `package_test` receipt are accepted on `development`; Step 9b.1's private container, isolated GnuPG runner and encrypt-once library are accepted on `development` (synthetic recipients only); Step 9b.2's ciphertext reuse, release binding receipt v1, absent production loader and scope separation are accepted on `development`; the remaining Step 9 custody and release gates remain staged. | | [agent-run-fixture.md](agent-run-fixture.md) | The `relavium agent run --fixture` cassette format — a `StreamChunk[][]` JSON recording of an agent run's LLM stream, for deterministic offline replay. | diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 8499700a..649e2eef 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -1,6 +1,6 @@ # CLI Packaging Contract -- **Status**: Compiled graph, installed package and Step 8 native activation independently accepted on `development`; the Step 9a.1 candidate build and finalizer are independently accepted on `development` (2026-10-24); Step 9a.2a's strict decoder, private manifest and symbolication are independently accepted on `development` (2026-10-24); Step 9a.2b's product map controls, post-receipt rebuild refusals and `package_test` receipt are independently accepted on `development` (2026-10-24); Step 9b.1's private container, isolated GnuPG runner and encrypt-once library are independently accepted on `development` (2026-10-25; synthetic recipients only); Step 9b.2's ciphertext reuse, release binding receipt v1, absent production loader and scope separation are implemented on `development` and under review (synthetic recipients only); the remaining Step 9 custody and release gates remain staged +- **Status**: Compiled graph, installed package and Step 8 native activation independently accepted on `development`; the Step 9a.1 candidate build and finalizer are independently accepted on `development` (2026-10-24); Step 9a.2a's strict decoder, private manifest and symbolication are independently accepted on `development` (2026-10-24); Step 9a.2b's product map controls, post-receipt rebuild refusals and `package_test` receipt are independently accepted on `development` (2026-10-24); Step 9b.1's private container, isolated GnuPG runner and encrypt-once library are independently accepted on `development` (2026-10-25; synthetic recipients only); Step 9b.2's ciphertext reuse, release binding receipt v1, absent production loader and scope separation are independently accepted on `development` (2026-10-26; synthetic recipients only); the remaining Step 9 custody and release gates remain staged - **Date**: 2026-10-15 - **Related**: [ADR-0111](../../decisions/0111-cli-meta-operations-use-a-lazy-packaged-runtime.md), [commands.md](commands.md), [release runbook](../../runbooks/release-a-surface.md) @@ -713,18 +713,18 @@ refuses every artifact. Its review and acceptance are recorded in the renames and removes nothing. Its module imports nothing that can start a process, so reuse makes no GnuPG call, and it cannot re-check the ciphertext's recipients: a named limit. - **Diagnostics.** Every refusal is a fixed `PACKED_CLI_CUSTODY_`. A host failure is `UNEXPECTED`, - except on a named path, which carries that path's code: `PLACEMENT` for the destination, - `CANDIDATE_ASSET`, `CIPHERTEXT_DRIFT` and `TARBALL_DRIFT` for the files read, and - `PRODUCTION_RECIPIENT_INVALID` while loading the recipient; inside encryption, `encryptOnce`'s own - codes apply. No path, key or receipt content surfaces. + except on a named path, which carries that path's code: `PLACEMENT` when the destination cannot be + checked or resolved (one that resolves but cannot be listed is `UNEXPECTED`), `CANDIDATE_ASSET`, + `CIPHERTEXT_DRIFT` and `TARBALL_DRIFT` for the files read, and `PRODUCTION_RECIPIENT_INVALID` while + loading the recipient; inside encryption, `encryptOnce`'s own codes apply. No path, key or receipt content surfaces. - **Where it runs.** `release-recipient.test.mjs`, `release-binding.test.mjs`, `release-custody.test.mjs` and `release-scope.test.mjs` join `test:acceptance-tools`, as the 9b.1 tests do; the custody test needs GnuPG ≥ 2.2.8. `release-scope.test.mjs` pins the separation statically: - no product or tooling code reaches a test fixture, and only the loader reads the fixture pins; - only the release gate mentions `custody.mjs`, the one tracked file of that name, anywhere in source, - workflow and package-script text, comments included, once JavaScript and percent escapes, - separators and case are normalized; a path built at run time from pieces is beyond any static - control; + workflow and package-script text, comments included, once `\x`, `\u`, `\u{…}` and percent escapes, + separators and case are normalized; a path built at run time from pieces, or spelled with another + escape, is beyond this control (a recorded limit); - the loader is never called with an argument, and its test seam appears in no product or tooling file (permanent controls); - the receipt module's import closure, read from every `from` or `import` specifier whatever the diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index cc0c7b01..09cac3e4 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -138,7 +138,7 @@ The [deferred register](deferred-tasks.md) retains its existing limits; closure **Wave 3 Steps 1–8 merged to `main` on 2026-10-18 as [PR #95](https://github.com/HodeTech/Relavium/pull/95), merge commit `6d2a7372` (22:44 UTC). Wave 3 itself is not complete: Steps 9–19 had not started at -that merge (Steps 9a.1, 9a.2a and 9a.2b are now accepted on `development`, the native-identity increment NI and Step 9b.1 are accepted too, Step 9b.2 is implemented and under review, and Steps 9c–19 have not started) and whole-wave acceptance +that merge (Steps 9a.1, 9a.2a and 9a.2b are now accepted on `development`, the native-identity increment NI and Steps 9b.1 and 9b.2 are accepted too, and Steps 9c–19 have not started) and whole-wave acceptance remains Step 19.** The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) joins the completed scope, every review attribution, the independent review's dispositions and the publication state observed at merge. It was the maintainer-authorized, scoped publication checkpoint. @@ -172,7 +172,7 @@ bar and round cap, as its medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, by the author alone). Its other findings are recorded in -[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is independently accepted on `development` at `cf97b7f1` (2026-10-24): its round 13 reported no blocker, high or medium finding and its security sitting was clean, and the round's low findings are recorded in deferred-tasks.md ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)); Step 9a.2 is split into 9a.2a and 9a.2b, and Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24) after three rounds, the third with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)); Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24) after two rounds, the second with no blocker, high or medium finding and no regression ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)), and the native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25) after one round with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)), its Windows clause open until three green advisory Windows runs; Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b-preflight-decisions--2026-10-25)), and Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is independently accepted on `development` at `147c2788` ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-acceptance--2026-10-25)), and Step 9b.2 (the release binding receipt v1, verified ciphertext reuse, the absent production recipient loader and scope separation) is implemented on `development` and under review ([checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b2-implementation-checkpoint--2026-10-25)); the substrate's +[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is independently accepted on `development` at `cf97b7f1` (2026-10-24): its round 13 reported no blocker, high or medium finding and its security sitting was clean, and the round's low findings are recorded in deferred-tasks.md ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)); Step 9a.2 is split into 9a.2a and 9a.2b, and Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24) after three rounds, the third with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)); Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24) after two rounds, the second with no blocker, high or medium finding and no regression ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)), and the native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25) after one round with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)), its Windows clause open until three green advisory Windows runs; Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b-preflight-decisions--2026-10-25)), and Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is independently accepted on `development` at `147c2788` ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-acceptance--2026-10-25)), and Step 9b.2 (the release binding receipt v1, verified ciphertext reuse, the absent production recipient loader and scope separation) is independently accepted on `development` at `563402e6` ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b2-acceptance--2026-10-26)), with Step 9c next; the substrate's 9a.1–9d increments carry a dedicated archive/crypto security sitting. Steps 10–19 follow the [increment table](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) and end with whole-wave acceptance at Step 19. `development` is pushed after each accepted increment; PR #95 no longer bounds the @@ -1141,7 +1141,7 @@ an ESLint selector rather than a list. ### Wave 3 — Clear the ground -**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Steps 9a.1, 9a.2a, 9a.2b, NI and 9b.1 accepted on `development` (2026-10-24/25), Step 9b.2 implemented and under review, Steps 9c–19 NOT STARTED — 2026-10-25.** +**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Steps 9a.1, 9a.2a, 9a.2b, NI, 9b.1 and 9b.2 accepted on `development` (2026-10-24/26), Steps 9c–19 NOT STARTED — 2026-10-26.** ADR-0110–0113, their linked contracts and the nineteen-increment plan were approved on 2026-10-16. [Steps 1–8 were independently accepted](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) on `development`, with whole Step 8 reviewed at `c83c8575`, and published as the scoped checkpoint diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index 4714da5d..85822142 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -2,7 +2,7 @@ > Status: Living -> Last updated: 2026-10-25 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Step 9a.1 is independently accepted on `development` with its recorded low findings below, Step 9a.2a is independently accepted on `development` with its recorded low findings below, Step 9a.2b is independently accepted on `development` with its recorded low findings below, the native-identity increment NI independently accepted on `development` with its recorded low findings below (its Windows clause open), Step 9b.1 independently accepted on `development` with its named limits and recorded low findings below, Step 9b.2 implemented and under review with its named limits and handoffs below, and Steps 9c–19 not started; the PR95 increment is closed on `development` under the maintainer's 2026-10-21 round cap). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's edge cases with the maintainer's 2026-10-21 acceptance bar and round cap and the 2026-10-22 rulings. Existing approved deferrals remain. +> Last updated: 2026-10-26 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Step 9a.1 is independently accepted on `development` with its recorded low findings below, Step 9a.2a is independently accepted on `development` with its recorded low findings below, Step 9a.2b is independently accepted on `development` with its recorded low findings below, the native-identity increment NI independently accepted on `development` with its recorded low findings below (its Windows clause open), Step 9b.1 independently accepted on `development` with its named limits and recorded low findings below, Step 9b.2 independently accepted on `development` with its named limits, handoffs and recorded low findings below, and Steps 9c–19 not started; the PR95 increment is closed on `development` under the maintainer's 2026-10-21 round cap). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's edge cases with the maintainer's 2026-10-21 acceptance bar and round cap and the 2026-10-22 rulings. Existing approved deferrals remain. > The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) records completed prerequisites; full M/N lifecycle and other scheduled work remain with their owners. > > Earlier triage: the Phase-2.6 rewrite triaged every open item, the 2026-07-19 full-project @@ -927,6 +927,42 @@ The release modules are library code with synthetic controls only: nothing outsi non-canonical `relavium-0.1.1.tgz.gpg` name is corrected with the guard. *(scheduled · tools/cli-package/release-custody.mjs, tools/cli-package/release-guard-core.mjs)* +## Wave 3 Step 9b.2 — recorded low findings (2026-10-26) + +Recorded with the Step 9b.2 acceptance after round 4 +([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b2-acceptance--2026-10-26)). +**Trigger for every entry:** Step 9d's update of `release-scope.test.mjs` and the release modules, or the next change +to the file it names. + +- [ ] **The `custody.mjs` control does not decode identity or legacy octal escapes.** *Low; round 4, PLAN-9B2-R4-1, + COR-R4-2.* `namesFile` decodes `\x`, `\u`, `\u{…}` and percent escapes, but a specifier such as + `'./custo\dy.mjs'` (an identity escape) or a sloppy-mode octal escape still loads `custody.mjs` unseen. The + spelling is deliberate, never an honest mistake. **Re-check:** also match a variant with each stray backslash + removed and octal escapes decoded, or refuse any backslash in a specifier. *(low · + tools/cli-package/release-scope.test.mjs)* +- [ ] **The static controls skip `tools/lint-fixtures/assert-fence.mjs`.** *Low; round 4, PLAN-9B2-R4-2.* The scan + treats all of `tools/lint-fixtures/` as test-only, but `assert-fence.mjs` is CI tooling (`pnpm lint:fence-check`), so + a `custody.mjs`, fixture or test-seam reference there passes the controls. **Re-check:** exempt only the quarantined + `.ts` fixtures and add the script to the scan's own assertion. *(low · tools/cli-package/release-scope.test.mjs)* +- [ ] **The closure's specifier scan has blind spellings.** *Low; round 4, SIT9B2-R4-L1, COR-R4-3, T-R4-3.* A comment + between `from` or `import` and the specifier, a specifier containing the other quote character, and computed member + access such as `process['getBuiltin' + 'Module']` each pass the closure control, which proves reuse starts no + process. Each spelling is deliberate, and today's closure contains none. **Re-check:** strip comments and tokenize + string literals properly before scanning, and refuse computed access on `process` in closure files. *(low · + tools/cli-package/release-scope.test.mjs)* +- [ ] **The loader reads the root by path after its link check.** *Low; round 4, COR-R4-4.* `lstat` then `readdir` + and the two reads leave a window in which a swapped root is read through a link and branded. Only a committed, + maintainer-owned directory is read in production. **Re-check:** open the root with `O_DIRECTORY | O_NOFOLLOW` and + compare its device and inode before and after the reads, as `readOrdinaryFile` does for files. *(low · + tools/cli-package/release-recipient.mjs)* +- [ ] **Some exact rules have no isolating test.** *Low; round 4, T-R4-1, T-R4-2, T-R4-4.* No history puts its current + entry before a retired one, so choosing the last entry would pass; the toolchain's prerelease suffix (its character + set and 64-character bound) and the target's digit count (three admitted, four refused) are unpinned; and a nit: + the sealed-link case asserts the first link's target, and the non-POSIX fallback of the linked-tarball case removes + the tarball instead. **Re-check:** add those cases on the next change to the release tests. *(low · + tools/cli-package/release-recipient.test.mjs, tools/cli-package/release-binding.test.mjs, + tools/cli-package/release-custody.test.mjs)* + ## Wave 3 NI — recorded low findings (2026-10-25) Round 1 of the native-identity increment at `fc5af701` accepted it with these lows diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 9a8ebef5..7dd35d71 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -3704,7 +3704,7 @@ No credential values or request captures belong in tracked evidence. | NI | **Native file identity on Windows (maintainer-approved 2026-10-24; after 9a.2b, before 9b):** every file and directory identity check compares `dev`/`ino` read with `{ bigint: true }` — the filesystem policy's pinned roots, the workflow catalog's open check and the media store's pins, read check and temporary claims — so a replaced file or directory can never compare equal on Windows, where `ino` is a 64-bit NTFS file ID that a JavaScript number cannot hold exactly; a replacement test per site. Found by 9a.2a's push CI ([acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)); recorded in [deferred-tasks.md](../deferred-tasks.md#native-file-identity-on-windows--approved-increment-2026-10-24). Security sitting. *Preflight decisions approved on 2026-10-25 ([decisions](#wave-3-ni-preflight-decisions--2026-10-25)); independently accepted on `development` at `fc5af701` on 2026-10-25 ([checkpoint](#wave-3-ni-implementation-checkpoint--2026-10-25), [acceptance](#wave-3-ni-acceptance--2026-10-25)); its Windows clause closes after three green advisory Windows runs on pushed heads.* | | 9b | **Recorded split — recipient custody:** synthetic-only isolated GnuPG controls (public-only home/keyring, pinned primary and encryption-subkey fingerprints, machine-readable refusal of missing, expired, revoked or unusable recipients), encrypt once and reuse the original ciphertext. Explicit crypto/custody security sitting. Fixture proof never claims production recipient or custody configuration. *Split on 2026-10-25 (maintainer-approved, L8) into 9b.1 and 9b.2 below, which replace it as increments ([preflight decisions](#wave-3-step-9b-preflight-decisions--2026-10-25)).* | | 9b.1 | **Recorded split, 2026-10-25 (maintainer-approved) — container, isolated GnuPG runner and encrypt-once:** the strict USTAR dialect extracted from the archive reader with its writer promoted; the private container built from the sealed staging in a staging-relative layout, each map's digest checked against the private manifest and its size against the finalizer's record, which must name exactly the manifest's outputs and maps (the metafile must be a JSON object); an isolated public-only GnuPG runner (GnuPG ≥ 2.2.8; `--recipient-file` with no keyring or agent, and no trustdb but the empty one GnuPG may create in the listing home); a colon-listing pre-check that refuses missing, expired, revoked, unusable (v4 with cv25519 or RSA ≥ 3072 only) or unexpected recipients and secret material; `encryptOnce` (SEIPDv1 with MDC and AES-256, exactly one `ENC_TO` for the pinned subkey, the exact produced asset, a no-clobber publish, plaintext cleanup on every path); committed synthetic public fixtures, an ephemeral round-trip key, and the GnuPG prerequisite documented. Library and synthetic controls only: product paths and workflows stay unchanged. Crypto/custody security sitting. *Independently accepted on `development` at `147c2788` (2026-10-25) ([decisions](#wave-3-step-9b-preflight-decisions--2026-10-25), [checkpoint](#wave-3-step-9b1-implementation-checkpoint--2026-10-25), [acceptance](#wave-3-step-9b1-acceptance--2026-10-25)).* | -| 9b.2 | **Recorded split, 2026-10-25 (maintainer-approved) — reuse, binding receipt and scope separation:** retained-ciphertext reuse without re-encryption (digest and size verified, zero GnuPG calls); the release binding receipt's schema v1, its validator and a sealer that writes only after the ciphertext exists (9d adds the origin, map-check and `ci`-run binding); the production recipient loader, absent in Wave 3; synthetic and production scope separation. Crypto/custody security sitting. *Implemented on `development` and under review ([decisions](#wave-3-step-9b2-preflight-decisions--2026-10-25), [checkpoint](#wave-3-step-9b2-implementation-checkpoint--2026-10-25)).* | +| 9b.2 | **Recorded split, 2026-10-25 (maintainer-approved) — reuse, binding receipt and scope separation:** retained-ciphertext reuse without re-encryption (digest and size verified, zero GnuPG calls); the release binding receipt's schema v1, its validator and a sealer that writes only after the ciphertext exists (9d adds the origin, map-check and `ci`-run binding); the production recipient loader, absent in Wave 3; synthetic and production scope separation. Crypto/custody security sitting. *Independently accepted on `development` at `563402e6` (2026-10-26) ([decisions](#wave-3-step-9b2-preflight-decisions--2026-10-25), [checkpoint](#wave-3-step-9b2-implementation-checkpoint--2026-10-25), [acceptance](#wave-3-step-9b2-acceptance--2026-10-26)).* | | 9c | **Recorded split — offline release state controller:** permission-enforcing offline GitHub/npm controller, permissioned prebuild discovery and reservation, and every rerun, partial, conflict and expiry state, with no overwrite, rebuild or re-encryption on uncertain history. | | 9d | **Recorded split — workflow, runbook and whole-Step-9 landing:** least-privilege actual workflow dependency graph, required artifact/platform aggregation and exact-artifact consumers, lost-staging refusal and the release runbook update. The whole-Step-9 cumulative review retains all 9a–9d changes with preservation traces into Step 8, and the dedicated archive/crypto security sitting closes the step. | | 10 | **CLI behaviour/discovery integration:** #45/#264/#265/#268/#13/#17/#18/#310/#309/#19 under ADR-0110, consuming Steps 6–9. Production owner category matrix, real root/subtree JSON help, pricing/chat aliases, non-echoing Commander refusal and signal-safe deletion consent. G15 is refuted and receives real engine/store/log preservation evidence, with no invented token coalescing. Repeat affected packed gates. | @@ -10580,3 +10580,38 @@ real gaps, the `typescript`, `tsup` and `esbuild` length bounds, and gained case observes. Of 40 named mutants, 38 are killed; the two survivors are the equivalents named above. + +### Wave 3 Step 9b.2 acceptance — 2026-10-26 + +**Step 9b.2 is independently accepted on `development` at `563402e6` (2026-10-26)** under the Acceptance row as +amended on 2026-10-23. An internal rigor pass and rounds 1–3 were not accepted, each with mediums corrected as the +[checkpoint](#wave-3-step-9b2-implementation-checkpoint--2026-10-25) records: +- the rigor pass: the loader's exported seams could brand any key, the import control matched three spellings, and + half the recipient pair was untested; +- round 1: the rigor fix's resolved destination dropped a 9b.1 refusal, stray destination entries led to an unsealable + ciphertext, and a bare specifier escaped the import control; +- round 2: a nested package script escaped the import control, and a trailing separator bypassed the link refusal; +- round 3: four untested paths or control gaps, in the link failure, a linked tarball, the trailing anchors and a + template literal. + +Round 4 was a NEW independent complete cumulative review of `819f8cbb..563402e6`, with every reviewer in its own +worktree detached at that head. It reported no blocker, high or medium finding and no regression. Its separate +crypto/custody security sitting, run at the same head as its own reviewer and report, found lows only. The +integration lens and the critic were clean. The plan lens confirmed the inventory pin at every commit and that no 9b.1 +module, product path, receipt, guard or workflow changed. + +**Author evidence at `563402e6`.** The full `pnpm run ci` passed plain and alone (exit 0): `test:acceptance-tools` +1,024 of 1,024, all 25 product map controls, and every causal-control family. On Ubuntu 24.04 with GnuPG 2.4.4 (in +Docker, a fresh clone, a non-root user), the 90 tests of the 9b.1 custody files and the four release files ran: 89 +passed and the case-insensitive-volume case skipped. + +The round's verified findings were twelve lows and one nit: +- Two are corrected in the acceptance commit, which changes no code. The contract's diagnostics bullet no longer + says every host failure on the destination is `PLACEMENT`: a destination that cannot be listed refuses `UNEXPECTED` + (PLAN-9B2-R4-3, SIT9B2-R4-L2, COR-R4-1). The `custody.mjs` control's claim now names the escapes it decodes + (PLAN-9B2-R4-1, COR-R4-2). +- The rest are recorded in + [deferred-tasks.md](../deferred-tasks.md#wave-3-step-9b2--recorded-low-findings-2026-10-26), each with its example + and re-check. + +Step 9c is next. From 64fb87722d9dd62c61d96d671cd3d5dfb69db72e Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Wed, 4 Nov 2026 10:03:36 +0300 Subject: [PATCH 88/97] docs(repo): reset the roadmap priority to the CLI MVP The maintainer paused Wave 3 after Step 9b.2 on 2026-10-26 and put the CLI MVP first; D58-D72 were ruled on 2026-11-04 from a multi-agent review of the roadmap documents and the code at 5a328bca. - current.md is reduced from 2,095 to 420 lines: live status, the MVP order (Gate 0, MVP-1...MVP-10, then v0.2.1), the after-MVP queue with Waves 3-7, the decision table with D58-D72, standing rules and history links. Every inbound anchor is kept. - Unique records move verbatim: the 2026-07-26 execution plan with Waves 0-1 and the PR #81 closing list (the #W15 definitions) to the Phase 2.5.5 register, the 2.6.F and 2.6.C implementation records to Phase 2.6, and the W4 start-gate ruling to Phase 2.6.5. Duplicated history is deleted. - Phase 2.5.5 records the priority reset: the pause, the evidence, the decisions, seven recorded Wave 3 splits with their order amendments, dated amendments to the Acceptance, Publication and interim release guard rulings, and NI's third green Windows run, which closes its clause. - Phase 2.6 gains the CLI MVP additions (MVP-A1...A23) as the one home for new items, the D62 direction on the default grant, the ADR row 9/11 splits and the M2.6-1 note. - The milestone spine gains M3.5 (CLI MVP, v0.2.0). deferred-tasks.md closes the NI entry and re-points CR-70, CR-71, #61 and the chat-resume entry. - The CLAUDE.md and AGENTS.md status blocks are condensed to the current state. Docs only; no source, test or workflow change. Refs: ADR-0111, ADR-0057, ADR-0095 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 119 +- CLAUDE.md | 200 +- docs/roadmap/README.md | 1 + docs/roadmap/current.md | 2157 ++--------------- docs/roadmap/deferred-tasks.md | 15 +- .../phase-2.5.5-hardening-and-remediation.md | 515 +++- .../phase-2.6-conversational-authoring.md | 127 +- ...hase-2.6.5-core-reliability-remediation.md | 4 + 8 files changed, 919 insertions(+), 2219 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a899dc5a..8ecf7add 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,116 +22,15 @@ Turborepo + pnpm monorepo (`packages/shared`, `packages/llm`, `packages/core`, **cloud** (BYOK-central, Phase 2), and **managed** (Relavium's own keys via a metered egress gateway; engine stays local, Phase 2) — split across build phase 5 (managed inference) and phase 6 (cloud execution + portal); the engine is identical across all three (ADR-0012..0015). -**Status: Phase 1 is complete; Phase 2 (CLI) is feature-complete (published as v0.1.1); -Phase 2.5 (CLI Consolidation) is complete (M2.5-4, 2026-07-08); Phase 2.6 -(Conversational Authoring and the First-Class CLI) is in progress, with an interlude — -Phase 2.6.5 (Core reliability remediation) — completed between remediation Waves 1 and 2: -49 of 51 items closed on `main`, with approved deferrals retained; W8 merged 2026-10-11 in PR #92 (`CR-96` and `CR-97` were opened 2026-09-14, `CR-98` on 2026-09-18; `W7` merged to `main` on 2026-10-10 (PR #90), with ADR-0095–ADR-0103 -accepted) — all eight P0 blockers (ADR-0078–ADR-0084, merged 2026-08-24), the `W2` -liveness-and-deadlines wave (ADR-0085, merged 2026-08-28 as PR #85), `W3` resource governance and bounds (ADR-0086), merged 2026-08-30 as PR #86 **with a live blocker and nine open findings at that merge** — the blocker is now closed by W8/PR #92; scoped residuals remain in docs/roadmap/deferred-tasks.md — and `W4`, the hostile MCP boundary (ADR-0088), merged 2026-09-01 as PR #87 — a systematic review of the branch found five merge blockers, all reproduced and fixed before it merged. `W5`, media correctness (ADR-0089 + ADR-0090), **merged 2026-09-02 as PR #88** — a systematic review of that branch returned six merge blockers, all reproduced and fixed before the merge. `W6`, authoring correctness (ADR-0091–ADR-0094), **MERGED 2026-09-04 as PR #89** — `merge_strategy: first` means first DECLARED, a widened tool grant is refused when the plan is BUILT, an expression sees only its transitive closure with a literal out-of-closure read refused at parse, and an `output_schema` is compiled at parse and enforced at run time with **no new dependency**. Seven internal review rounds found that most defects were in the FIXES rather than the code they repaired; a systematic maintainer review then returned two merge blockers, both breaking a headline claim of the wave, plus eight further High findings — all reproduced and fixed before the merge. Each is a dated correction inside its own ADR. -**W7 merged to `main` on 2026-10-10 as PR #90, including final whole-wave Step 12.** -Steps 1–11, all five genuine overflow records, ADR-0102 request ownership and consolidated ADR-0103 -host integration have independent acceptance. ADR-0095–ADR-0103 are Accepted. All six W7 register -items are closed (47/51 at the W7 merge); PR #90 is merged (`46a6a701`); W7 has no open implementation or review gate. The -[closing register](docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md#w7-closing-register--2026-10-10) -joins evidence and every submitted review disposition. W8 subsequently merged in PR #92; approved deferrals remain. -No user decision, billing setup, credential or provider capture is pending. -**W8 merged to `main` on 2026-10-11 as PR #92 (`ef0ffa54`); Phase 2.6.5 is complete with approved deferrals.** -All seven steps, CR-81/82 and ADR-0087 §1 are complete on `main`; the register is **49/51**. -The [merge checkpoint](docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md#w8-merge-checkpoint--2026-10-11) -joins original whole-wave acceptance, maintainer corrections and final reviewed-head `3d4491df` -PR/push CI and SonarCloud success. Published ordinary/enforced coverage records 511 files / -11,026 passing tests / 12 established skips; all four wire and 23 usage mechanisms pass. -CR-70/93, CR-95's long continuation and exact-capacity/scoped residuals remain approved deferrals. -The joint [Wave 2 — Shut the doors](docs/roadmap/current.md#wave-2-merged--shut-the-doors) merged to `main` on 2026-10-15 as PR #94; -next is [Wave 3 — Clear the ground](docs/roadmap/current.md#wave-3--clear-the-ground). -Phase 2.5.5 and Phase 2.6 remain in progress. No W8 implementation or landing gate remains. -**Post-closure corrections are independently accepted, 2026-10-10:** core, CLI/DB and tooling/docs -each have two NEW clean complete cumulative review rounds. The supplemental publication-tool -correction also has two NEW complete clean rounds. The [post-closure register](docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md#post-closure-systematic-review--2026-10-10) -joins every report disposition, new permanent/causal evidence and current gates. Published-head checks -passed at reviewed head `69ac80b0` before PR #90 merged. No W7 implementation, provider, -billing, key or user decision is pending. The [merge checkpoint](docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md#w7-merge-checkpoint--2026-10-10) -records W7 closure. W8 subsequently merged in PR #92; the register is 49/51 on `main`. -Existing approved limits are unchanged. - -**PR #92 maintainer corrections:** the [correction register](docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md#w8-pr-92-systematic-review-corrections--2026-10-11) -records all four increments and the subsequent publication correction accepted after two NEW -clean complete cumulative rounds each. Fresh full CI and bare/enforced coverage pass at `6139563e`, -including four wire and 23 usage controls. -Final reviewed head `3d4491df` passed PR/push CI and SonarCloud before PR #92 merged. -No merge-commit execution or new whole-PR review is inferred. Two nonblocking Sonar complexity -suggestions remain visible; no new deferral or decision is added. - -**Joint Wave 2 merged to `main` — 2026-10-15, PR #94 (`ab08209e`).** -All eight implementation steps and the maintainer corrections are complete on `main` -under Accepted ADR-0105–0109. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) -joins forty item dispositions, security sittings, the final reviewed source `4cdfe011`, -two NEW clean complete cumulative correction rounds and successful PR/push CI, -coverage, native-platform and independently inspected physical controls. Earlier -whole-wave and correction reviews retain their exact source scopes; no new whole-PR -review or merge-commit execution is inferred. Detailed review records remain local -and Git-ignored. - -Exit criteria 1–3 and the D half of M2.5.5-2 are satisfied on `main`. The I half, -whole Phase 2.5.5/2.6, full M/N production lifecycle, future live provider headers -and existing approved deferrals remain open in their canonical homes. No Wave 2 -implementation, review, publication, ADR approval, provider capture, credential, -billing or user-decision gate remains. Next is [Wave 3 — Clear the ground](docs/roadmap/current.md#wave-3--clear-the-ground), -whose ADR-0110–0113/contracts/nineteen-increment plan were approved on 2026-10-16. -Its serialized multi-file Step 1 is independently accepted on `development` -(2026-10-16); Step 2 is also accepted after two NEW clean complete cumulative rounds -following its historical-reader corrections. Step 3 (ADR-0113 secret provenance) -is independently accepted after two NEW complete cumulative clean rounds and a -separate clean security sitting. Step 4 (error/envelope substrate) is independently -accepted after two NEW complete cumulative clean rounds. Step 5 (tools and engine -tails) is independently accepted after two NEW complete cumulative clean rounds -following its registry-documentation correction and a separate clean security -sitting. Step 6 (pure process/discovery substrate) is independently accepted after -two NEW complete cumulative clean rounds and a separate clean security sitting. -Step 7a (lazy runtime and finite compiled graph) is independently accepted after -two NEW complete cumulative clean rounds and a separate clean security sitting. -Step 7b and whole Step 7 are independently accepted at `c836ffe6` after two NEW -complete cumulative clean rounds, a clean security sitting and successful exact-tarball -Linux/macOS/Windows plus Node 22.0.0 consumers. Broader CI retains its separately -attributed status. Step 8a implements origin adapters and an inert native foundation; -it is independently accepted at `e0a5ae19` after two NEW complete cumulative -clean rounds and a separate clean security sitting. -Step 8b (creation-owned native processes) is independently accepted at -`e16a7659` after two NEW complete cumulative clean rounds and a separate clean -security sitting following its canonical-reference correction. Its unchanged -runtime/control evidence and successful full CI plus all four same-tarball -platform consumers remain attributed to `ab50be11`, with independently verified -source/dependency equivalence. Step 8c workflow/preparation/terminal integration -is independently accepted at `dbab25d8` after two NEW complete cumulative clean -rounds, a separate clean security sitting and successful full CI. All four -same-tarball platform consumers pass at that reviewed source. -**Step 8d and whole Step 8 are independently accepted on `development` at -`c83c8575` — 2026-10-17.** Two NEW complete cumulative CLEAN rounds (R25/R26) -each cover all 342 paths / 1,842 ordered hunks after the final fixture correction. -A NEW separate clean security sitting and all fifteen full-CI jobs pass at that -reviewed source, as do all four actual same-tarball platform consumers. Original -failed reviewer-preparation records and source/native/platform limits remain -explicit; source CI does not certify later documentation commits. The -[acceptance register](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) -joins exact evidence and historical corrections. Steps 1–8 are accepted; -**Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 -itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now -accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI and Steps 9b.1 and 9b.2 are accepted too, and Steps 9c–19 have not started) and whole-wave acceptance remains Step 19. -The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) -records what merged and its review attributions. The final head `a3e164b5` (corrections from a -further independent review, including a secret-scrubbing rule) merged with author verification -only and was then independently reviewed (four rounds and a security sitting, with follow-up -fixes on `development`, not yet on `main`); CI for it and the merge commit finished green, and SonarCloud's `main` -gate (red since at least PR #90) was cleared on 2026-10-19. A pre-implementation review of Steps 9–19 -and the maintainer's [2026-10-19 rulings](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) -give every open PR #95 finding an owner and record 29 increments (no new ADR). The PR95 increment -(the scrubber residuals, including both security residuals) is closed on `development` under the -maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were -NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last -round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25): round 1 reported no blocker, high or medium finding and no regression, with a clean security sitting; every product file and directory identity check now compares exact bigints, its low findings are recorded in deferred-tasks.md, and its Windows clause closes after three green advisory Windows runs ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)). Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b-preflight-decisions--2026-10-25)); Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is independently accepted on `development` at `147c2788` (2026-10-25): rounds 1–3 were not accepted (one, one and two mediums, each corrected), and round 4 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-acceptance--2026-10-25)). Step 9b.2 (the release binding receipt v1, verified ciphertext reuse, the production recipient loader, absent in Wave 3, and scope separation; synthetic recipients only) is independently accepted on `development` at `563402e6` (2026-10-26): an internal rigor pass and rounds 1–3 were not accepted (mediums each time, all corrected), and round 4 reported no blocker, high or medium finding and no regression, its security sitting lows only; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b2-acceptance--2026-10-26)). Step 9c is next. `development` is pushed after each accepted increment. No provider, -credential, billing or new ADR approval is pending for the approved implementation plan. +**Status (2026-11-04): the CLI MVP comes first.** Phases 1, 2 and 2.5 are complete (`relavium@0.1.1` is +published); Phase 2.6.5 is complete with approved deferrals (49/51, PR #92). Phase 2.5.5's Waves 0–2 and Wave 3 +Steps 1–8 are merged (PR #94, PR #95), and Steps 9a.1–9b.2 and NI are accepted on `development`. On 2026-10-26 +the maintainer paused Wave 3 (Steps 9c–19 pending, not declined) and put the CLI MVP first: `relavium` v0.2.0 +(milestone M3.5), ordered as MVP-1…MVP-10 under decisions D58–D72 in +[current.md](docs/roadmap/current.md#next-work--the-cli-mvp). Phase 2.6 is in progress (2.6.C and 2.6.F done, +2.6.Q P1–P5 landed). The interim release guard blocks npm publication until the pre-1.0 release ADR (D59) is +Accepted; the acceptance bar is tiered (D64), with security-touching increments keeping a full review round and +a separate security sitting. CLAUDE.md carries the full summary. See [docs/roadmap/current.md](docs/roadmap/current.md) for live status. diff --git a/CLAUDE.md b/CLAUDE.md index 68cdba2d..de059aaa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -39,188 +39,24 @@ A run executes in one of **three execution modes** behind the one `LLMProvider` engine is identical across all three. See [ADR-0012](docs/decisions/0012-managed-inference-dual-mode.md) to [ADR-0015](docs/decisions/0015-managed-mode-data-handling-and-compliance.md) and [docs/architecture/managed-inference.md](docs/architecture/managed-inference.md). -**Status: Phase 1 is complete** (2026-06-21) — the engine runs end-to-end behind the -`LLMProvider` seam with all three adapters, `AgentSession`, multimodal I/O, and the full -run-loop. **Phase 2 (CLI) is feature-complete** (published as v0.1.1): -`relavium chat` / `chat-resume` / `agent run`, `relavium run` with `--json`, the -inbound MCP client, and the full YAML lifecycle (`create`/`import`/`export`). -**Phase 2.5 (CLI Consolidation) is complete** (M2.5-4, PR #69, 2026-07-08): the bare-invocation -Home, the two-registry command system (`/` palette + shell commands), per-tool -approval/modes (ask/plan/accept-edits/auto), `@`-mention file injection, context -compaction, the onboarding wizard + live model catalog, reasoning rendering, and regression -hardening. **Phase 2.6 (Conversational Authoring and the First-Class CLI) is in progress** — two -workstreams are merged to `main`: **2.6.F** (the platform floor + the full-screen TUI renderer, -PR #74, 2026-07-11) and **2.6.C** (the reseat transcript-carry + the `/cost` per-model breakdown, -PR #75, 2026-07-13). The phase is a full-screen Home-managed CLI with conversational workflow authoring, management -browsers, competitor-breadth tools, settings/theming/`en`+`tr` localization, and the -run-ops resume follow-up. - -**The interlude between remediation Waves 1 and 2 is complete: Phase 2.6.5 (Core reliability), -49 of 51 items closed on `main`, with approved CR-70/93 and scoped deferrals retained. -W8 merged on 2026-10-11 as PR #92; W7 had restored the non-deferrable criterion** — `W0` (PR #82, 2026-08-11), `W1`, the eight P0 blockers plus `CR-92`, merged -2026-08-24 (PR #83) behind [ADR-0078](docs/decisions/0078-ordered-durable-append-and-the-terminal-outbox.md)–[ADR-0084](docs/decisions/0084-consent-before-a-local-mcp-spawn.md): -ordered durable append, cross-process run ownership, the durable effect journal, untrusted compaction summaries, -the stream-grammar seam obligation, engine-side input admission and resume identity, and consent before a local -MCP spawn. **`W2` — liveness and deadlines — merged 2026-08-28 (PR #85)** behind -[ADR-0085](docs/decisions/0085-the-node-executor-owes-liveness-and-the-engine-enforces-it.md): the authored -agent `timeout_ms` now bounds the node, every media call is bounded, a resume re-arms the run cap and a -pending gate at their REMAINING time rather than renewing them (the node bound is the deliberate exception — -ADR-0085 §2 records why), and an executor that ignores its signal can no longer leave a run without a -terminal. **`W3` — resource governance and bounds — merged 2026-08-30 (PR #86) behind -[ADR-0086](docs/decisions/0086-absolute-admission-ceilings-on-authored-values.md)**: authored values carry -absolute admission ceilings, node output / workflow state / durable events are size-bounded, and a long-lived -engine no longer retains every finished run. **It merged with a live blocker** — an un-pulled event stream -drops the terminal event, breaching ADR-0036's gap-free contract — plus nine verified findings, all recorded -in the `W3` residuals of [deferred-tasks.md](docs/roadmap/deferred-tasks.md). -[ADR-0087](docs/decisions/0087-consumed-streams-size-bounds-and-run-retention.md) records the fix and, after a -review that refused acceptance until nine dated corrections existed, was **Accepted (2026-09-04) with its §1 -unimplemented**. That blocker is now closed by W8/PR #92. Its §3 turned out to be implemented already, in `W5`. **`W4` — the hostile MCP boundary — merged 2026-09-01 (PR #87) -behind [ADR-0088](docs/decisions/0088-the-mcp-boundary-is-hostile.md)**: every MCP call is bounded and -cancellable, `http`/`sse` connect by validated pinned IP, a redirect is refused, a remote `websocket` is -refused at admission, a server's ingress is bounded at two levels, and its tool DEFINITIONS are treated as -untrusted. No new product surface in either wave — only the invariants an existing surface already claims. -A systematic review of PR #87 found **five merge blockers** in `W4` — all reproduced and fixed before it -merged; the phase doc's `W4` section records what they were and what the pattern says. **`W5` — media correctness — MERGED 2026-09-02 (PR #88)** behind -[ADR-0089](docs/decisions/0089-media-correctness-four-boundaries.md) and -[ADR-0090](docs/decisions/0090-a-continuation-token-rides-the-part-it-belongs-to.md): `read_media` delivers -over the media-input rail on a marked, fenced synthesized message; tool and attachment capability is gated on -the MODEL; Gemini's function-call continuation token rides the part it belongs to; a media body streams from -the network into the store under a size ceiling, an idle deadline and the run signal; a `url` output is pinned -to a content-addressed handle at first resolution; and a missing media rate is unpriced rather than a price of -zero. A systematic maintainer review of the branch returned **six merge blockers**, all reproduced and fixed -before it merged. Two items closed one half of a two-part obligation and say so; every residual is written out -rather than left inside a checked box. **`W6` — authoring correctness — MERGED 2026-09-04 (PR #89)** behind [ADR-0091](docs/decisions/0091-first-means-first-declared-not-first-to-finish.md)–[ADR-0094](docs/decisions/0094-a-tool-grant-is-checked-when-the-plan-is-built.md): -`merge_strategy: first` means first DECLARED and the plan field no longer claims otherwise; a widened tool grant -is refused when the plan is BUILT rather than mid-run; an expression sees only its transitive dependency closure -and a literal out-of-closure `run.outputs` read is refused at parse (as is `edges[].condition`, which nothing -ever read); and an authored `output_schema` is compiled at parse in an allowlist-strict mode and enforced at run -time — with **no new dependency**, because the JSON-Schema→Zod compiler the deferral claimed we needed already -existed in `packages/mcp` and simply moved to `@relavium/shared`. Seven internal review rounds found that **most defects -were in the FIXES rather than the code they repaired** — the branch-order search was wrong three times, the -expression scan's no-false-refusal claim was false in eight ways, and a stand-down gate silently removed the -check it was added to protect; each is a dated correction inside its own ADR. A systematic maintainer review -then returned **two merge blockers**, both breaking a headline claim of the wave — `required` was not enforced -when a property's schema constrained nothing, and the expression scanner's regex was quadratic and synchronous -on a parser that accepts 2 MiB — plus eight further High findings, all reproduced and fixed before the merge. - - -**Between `W6` and `W7`, the two remaining non-deferrable items closed (2026-09-06)**: `CR-73` — `invoke_agent` -was advertised to every model granted it while `ctx.invokeAgent` is wired nowhere in the tree, so the -advertise-filter now checks a tool's declared dispatch DELEGATE and not only its `ToolHost` arm (`read_media` -rides along, for the same reason and by `CR-50`'s own argument) — and `CR-80` — a rejected custom provider -`base_url` was caught and the DEFAULT adapter left standing, which is the official API, so a drifted config -silently sent prompts and keys to `api.openai.com`; it now installs a refusing adapter naming the URL's shape, -never its credentials. The test that was supposed to pin `CR-80` asserted only `toBeDefined()`, which a -refusing adapter satisfies too, so it passed either way — the finding under the defect was the test. -[ADR-0087](docs/decisions/0087-consumed-streams-size-bounds-and-run-retention.md) was also **accepted -(2026-09-04) with nine dated corrections** after a review refused it as written; §1 was still unimplemented -at that checkpoint, while §3 turned out to have shipped in `W5` under a different item number. -**W8 merged to `main` on 2026-10-11 as PR #92 (`ef0ffa54`); Phase 2.6.5 is complete with approved deferrals.** -All seven steps, CR-81/82 and ADR-0087 §1 are complete on `main`; the register is **49/51**. -The [merge checkpoint](docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md#w8-merge-checkpoint--2026-10-11) -joins the original whole-wave acceptance, all maintainer-correction dispositions and final reviewed-head -`3d4491df` PR/push CI plus SonarCloud success. The ordinary/enforced-coverage inventory is -511 files / 11,026 passing tests / 12 established skips; four wire and 23 usage mechanisms pass. -Every correction increment has two NEW independent complete clean cumulative rounds after its final fix; -scoped correction reviews are not a new whole-PR certification. Fresh full local CI and bare/enforced -coverage at `6139563e` retain their separate source attribution. No merge-commit execution is inferred. -Two nonblocking Sonar complexity suggestions remain visible. Historical counts and pending states -throughout the registers describe their original checkpoints. - -CR-70 (cross-turn tool-history carrying), CR-93 (tenant-scoped mutable state), CR-95's long continuation -and existing exact-capacity/scoped residuals retain their approved limits in -[deferred-tasks.md](docs/roadmap/deferred-tasks.md). No new deferral, provider capture, billing, -credential or W8 user decision is pending. The joint [Wave 2 — Shut the doors](docs/roadmap/current.md#wave-2-merged--shut-the-doors) merged to `main` on 2026-10-15 as PR #94; -next is [Wave 3 — Clear the ground](docs/roadmap/current.md#wave-3--clear-the-ground). -Phase 2.5.5 and Phase 2.6 remain in progress. - -**W7 merged to `main` on 2026-10-10 as PR #90, including final whole-wave Step 12.** -Steps 1–11, all five genuine overflow records, owned measured requests and consolidated provider/MCP/host -lifetimes have independent acceptance. ADR-0095–ADR-0103 are Accepted. The six W7 register items are closed -(47/51 at the W7 merge); PR #90 is merged (`46a6a701`); W7 has no open implementation or review gate. The -[closing register](docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md#w7-closing-register--2026-10-10) -joins the accepted scopes, causal controls and systematic-review dispositions. Existing approved deferrals -stay in [deferred-tasks.md](docs/roadmap/deferred-tasks.md); W8 subsequently merged in PR #92 and closed the phase with approved deferrals. No user decision, -billing setup, credential or provider capture is pending. - -**Post-closure corrections are independently accepted, 2026-10-10:** core, CLI/DB and tooling/docs -each have two NEW clean complete cumulative review rounds. The supplemental publication-tool -correction also has two NEW complete clean rounds. The [post-closure register](docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md#post-closure-systematic-review--2026-10-10) -joins every report disposition, new permanent/causal evidence and current gates. Published-head checks -passed at reviewed head `69ac80b0` before PR #90 merged. No W7 implementation, provider, -billing, key or user decision is pending. The [merge checkpoint](docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md#w7-merge-checkpoint--2026-10-10) -records W7 closure. W8 subsequently merged in PR #92; the register is 49/51 on `main`. -Existing approved limits are unchanged; W8 has no open implementation or landing gate. - -**Joint Wave 2 merged to `main` — 2026-10-15, PR #94 (`ab08209e`).** -All eight implementation steps and the maintainer corrections are complete on `main` -under Accepted ADR-0105–0109. The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) -joins forty item dispositions, security sittings, the final reviewed source `4cdfe011`, -two NEW clean complete cumulative correction rounds and successful PR/push CI, -coverage, native-platform and independently inspected physical controls. Earlier -whole-wave and correction reviews retain their exact source scopes; no new whole-PR -review or merge-commit execution is inferred. Detailed review records remain local -and Git-ignored. - -Exit criteria 1–3 and the D half of M2.5.5-2 are satisfied on `main`. The I half, -whole Phase 2.5.5/2.6, full M/N production lifecycle, future live provider headers -and existing approved deferrals remain open in their canonical homes. No Wave 2 -implementation, review, publication, ADR approval, provider capture, credential, -billing or user-decision gate remains. Next is [Wave 3 — Clear the ground](docs/roadmap/current.md#wave-3--clear-the-ground), -whose ADR-0110–0113/contracts/nineteen-increment plan were approved on 2026-10-16. -Its serialized multi-file Step 1 is independently accepted on `development` -(2026-10-16); Step 2 is also accepted after two NEW clean complete cumulative rounds -following its historical-reader corrections. Step 3 (ADR-0113 secret provenance) -is independently accepted after two NEW complete cumulative clean rounds and a -separate clean security sitting. Step 4 (error/envelope substrate) is independently -accepted after two NEW complete cumulative clean rounds. Step 5 (tools and engine -tails) is independently accepted after two NEW complete cumulative clean rounds -following its registry-documentation correction and a separate clean security -sitting. Step 6 (pure process/discovery substrate) is independently accepted after -two NEW complete cumulative clean rounds and a separate clean security sitting. -Step 7a (lazy runtime and finite compiled graph) is independently accepted after -two NEW complete cumulative clean rounds and a separate clean security sitting. -Step 7b and whole Step 7 are independently accepted at `c836ffe6` after two NEW -complete cumulative clean rounds, a clean security sitting and successful exact-tarball -Linux/macOS/Windows plus Node 22.0.0 consumers. Broader CI retains its separately -attributed status. Step 8a implements origin adapters and an inert native foundation; -it is independently accepted at `e0a5ae19` after two NEW complete cumulative -clean rounds and a separate clean security sitting. -Step 8b (creation-owned native processes) is independently accepted at -`e16a7659` after two NEW complete cumulative clean rounds and a separate clean -security sitting following its canonical-reference correction. Its unchanged -runtime/control evidence and successful full CI plus all four same-tarball -platform consumers remain attributed to `ab50be11`, with independently verified -source/dependency equivalence. Step 8c workflow/preparation/terminal integration -is independently accepted at `dbab25d8` after two NEW complete cumulative clean -rounds, a separate clean security sitting and successful full CI. All four -same-tarball platform consumers pass at that reviewed source. -**Step 8d and whole Step 8 are independently accepted on `development` at -`c83c8575` — 2026-10-17.** Two NEW complete cumulative CLEAN rounds (R25/R26) -each cover all 342 paths / 1,842 ordered hunks after the final fixture correction. -A NEW separate clean security sitting and all fifteen full-CI jobs pass at that -reviewed source, as do all four actual same-tarball platform consumers. Original -failed reviewer-preparation records and source/native/platform limits remain -explicit; source CI does not certify later documentation commits. The -[acceptance register](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) -joins exact evidence and historical corrections. Steps 1–8 are accepted; -**Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 -itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now -accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI and Steps 9b.1 and 9b.2 are accepted too, and Steps 9c–19 have not started) and whole-wave acceptance remains Step 19. -The [merge checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) -records what merged and its review attributions. The final head `a3e164b5` (corrections from a -further independent review, including a secret-scrubbing rule) merged with author verification -only and was then independently reviewed (four rounds and a security sitting, with follow-up -fixes on `development`, not yet on `main`); CI for it and the merge commit finished green, and SonarCloud's `main` -gate (red since at least PR #90) was cleared on 2026-10-19. A pre-implementation review of Steps 9–19 -and the maintainer's [2026-10-19 rulings](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) -give every open PR #95 finding an owner and record 29 increments (no new ADR). The PR95 increment -(the scrubber residuals, including both security residuals) is closed on `development` under the -maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were -NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last -round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, -by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25): round 1 reported no blocker, high or medium finding and no regression, with a clean security sitting; every product file and directory identity check now compares exact bigints, its low findings are recorded in deferred-tasks.md, and its Windows clause closes after three green advisory Windows runs ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)). Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b-preflight-decisions--2026-10-25)); Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is independently accepted on `development` at `147c2788` (2026-10-25): rounds 1–3 were not accepted (one, one and two mediums, each corrected), and round 4 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-acceptance--2026-10-25)). Step 9b.2 (the release binding receipt v1, verified ciphertext reuse, the production recipient loader, absent in Wave 3, and scope separation; synthetic recipients only) is independently accepted on `development` at `563402e6` (2026-10-26): an internal rigor pass and rounds 1–3 were not accepted (mediums each time, all corrected), and round 4 reported no blocker, high or medium finding and no regression, its security sitting lows only; its low findings are recorded in deferred-tasks.md ([acceptance](docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b2-acceptance--2026-10-26)). Step 9c is next. `development` is pushed after each accepted increment. No provider, -credential, billing or new ADR approval is pending for the approved implementation plan. +**Status (2026-11-04): the CLI MVP comes first.** Phase 1 (engine and LLM) is complete; Phase 2 +(CLI) is feature-complete and `relavium@0.1.1` is the published version; Phase 2.5 (CLI Consolidation) is +complete (M2.5-4, PR #69, 2026-07-08); Phase 2.6.5 (Core reliability) is complete with approved deferrals +(49/51, PR #92, 2026-10-11). Phase 2.5.5 (Hardening and Remediation): Waves 0–2 are merged (Wave 2: PR #94), +Wave 3 Steps 1–8 are merged (PR #95, 2026-10-18), and PR95, 9a.1, 9a.2a, 9a.2b, NI, 9b.1 and 9b.2 are +accepted on `development`. **On 2026-10-26 the maintainer paused Wave 3 — Steps 9c–19 are pending, not +declined — and put the CLI MVP first:** `relavium` v0.2.0 on npm (milestone M3.5), ordered as MVP-1…MVP-10 in +[current.md](docs/roadmap/current.md#next-work--the-cli-mvp) under decisions D58–D72 (ruled 2026-11-04). +Phase 2.6 (Conversational Authoring and the First-Class CLI) is in progress: 2.6.C and 2.6.F are done +(2.6.F's v0.2.0 release clause closes with the MVP) and 2.6.Q P1–P5 landed; the MVP takes slices of the rest. + +Two rules shape every MVP increment. Step 9a.1's interim release guard on `development` refuses any npm +publication until the superseding pre-1.0 release ADR (D59) is Accepted. The acceptance bar is tiered (D64): +an increment touching an approval or consent path, egress, keys or secrets, filesystem writes, a new terminal +sink for untrusted content, the lease or resume paths, or release keeps a rigor pass, one NEW complete +cumulative round with no blocker, high or medium finding, and a separate security sitting; other UX, +rendering and command increments are reviewed per milestone. `development` is pushed after each accepted +increment. For live status, per-PR history, milestone dates, and open obligations, see the canonical home [docs/roadmap/current.md](docs/roadmap/current.md); diff --git a/docs/roadmap/README.md b/docs/roadmap/README.md index c5cb6698..2fa4ddbf 100644 --- a/docs/roadmap/README.md +++ b/docs/roadmap/README.md @@ -149,6 +149,7 @@ to that phase's work breakdown). | **M1 ✅** | **LLM seam proven** *(achieved 2026-06-07, PR #9)* — `@relavium/llm`'s provider-agnostic `LLMProvider` seam with 3 adapters (Anthropic, the OpenAI-compatible adapter serving OpenAI + DeepSeek, Gemini), all passing one shared conformance suite (fixtures on PR, live nightly reserved/pending keys); no vendor SDK type crosses the seam. The `FallbackChain` runner (1.K, PR #13) and the cost tracker (1.B) have since landed, completing 1.m2. (Built **first** per the engine-first order — the engine's `AgentRunner` depends on it.) | 1 | Phase 1 · `@relavium/llm` seam + 3 adapters — conformance green at **1.J** (1.G–1.J; FallbackChain + cost tracker at 1.m2 / 1.B, 1.K) | | **M2 ✅** | **Engine end-to-end (critical-path milestone)** *(achieved 2026-06-16, PR #27)* — `@relavium/core` parses YAML→DAG and runs it (the `WorkflowEngine` + `AgentRunner` over all node types, emitting canonical events through the `RunEventBus`, with checkpoint/resume, retry, the deterministic expression sandbox, and the pre-egress budget governor), with the M1 seam wired in; the 1.U Node harness runs a workflow end-to-end with live streaming, checkpoint + cross-process resume, retry, and a provider fallback. Zero platform-specific imports. | 1 | Phase 1 · `@relavium/core` parser + runner + checkpoint/resume + retry + **expression sandbox (1.AB) + budget governor (1.AC)** + Node harness (1.L–1.U, all landed); the agent-first sub-spine (1.V–1.AA) lands in parallel (1.m5) | | **M3 ✅** | **CLI + engine regression harness** *(achieved 2026-06-23, PR #43)* — the `relavium` CLI drives the engine end-to-end with a live `ink` TUI and a deterministic `--json` CI mode; a small fixture suite runs in CI on every engine change as the agreed regression gate for Phases 2.5–6. (Phase 2 shipped complete as **v0.1.1**, PR #58; the CLI consolidation/authoring work continues in Phases 2.5–2.6 before M4.) | 1 | Phase 2 · 2.D (run wiring), 2.F (`--json` / CI mode), 2.K (regression harness) | +| **M3.5** | **CLI MVP — `relavium` v0.2.0 on npm** *(set 2026-11-04, decision D58)* — a new user installs the CLI, onboards with any of the four providers, chats with an agent that reads, searches, edits and runs commands under approval with diffs, authors workflows conversationally, runs them and sees their outputs, manages runs, sessions, providers and MCP servers from the Home, and reaches the web under the chat egress authority. Ordered as MVP-1…MVP-10 in [current.md](current.md#next-work--the-cli-mvp); it closes 2.6.F's release clause and M2.6-1, and takes slices of Phase 2.6 and the paused Wave 3 of Phase 2.5.5 without closing either phase. | 1 | Phase 2.6 · slices of 2.6.A/B/D/E/G/H/I/J/K/M/Q + the [CLI MVP additions](phases/phase-2.6-conversational-authoring.md#cli-mvp-additions--2026-11-04); Phase 2.5.5 · recorded Wave 3 splits | | **M2.5.5** | **Hardening & remediation review closed** — the 373-finding 2026-07-19 full-project review's verified defects against the Phase 0–2.5 codebase (and the merged 2.6.C/2.6.F slices) are resolved or explicitly re-deferred; the three CRITICAL findings (an unredacted tool-approval preview, a non-atomic chat-turn persister write, unsanitized `relavium run` terminal output) are closed. Runs **concurrently with Phase 2.6** — not a gate on M4/Phase 3, and not itself gated by any 2.6 workstream. | 1 | Phase 2.5.5 · 2.5.5.A–2.5.5.I | | **M4** | **Desktop agent-management center** — a signed, offline, no-account Tauri v2 app: ReactFlow canvas with all node types, live execution theater, gate overlay, local SQLite history (trace/Gantt/replay/retry-from-node), OS keychain; ADR-0010 perf gate + per-platform e2e green. | 1 | Phase 3 · 3.K (packaging), 3.L (e2e + perf gate), 3.P (sign-off) | | **M5** | **Product Phase 1 complete** — the standalone VS Code extension is shipped to the Marketplace: right-click run, live monitoring, human-gate webview, **no desktop app required**, same engine + events + SQLite history as the other surfaces. | 1 | Phase 4 · 4.E (right-click run), 4.H (gate webview), 4.K (Marketplace publish) | diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index 09cac3e4..81de18f7 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -1,1217 +1,215 @@ # Current state > Status: Living -> Last updated: 2026-10-25 +> Last updated: 2026-11-04 -- **Related**: [README.md](README.md), [phases/phase-2.5-cli-consolidation.md](phases/phase-2.5-cli-consolidation.md), [phases/phase-2.5.5-hardening-and-remediation.md](phases/phase-2.5.5-hardening-and-remediation.md), [phases/phase-2-cli.md](phases/phase-2-cli.md), [deferred-tasks.md](deferred-tasks.md), [../project-structure.md](../project-structure.md), [../tech-stack.md](../tech-stack.md) +- **Related**: [README.md](README.md), [phases/phase-2.6-conversational-authoring.md](phases/phase-2.6-conversational-authoring.md), [phases/phase-2.5.5-hardening-and-remediation.md](phases/phase-2.5.5-hardening-and-remediation.md), [phases/phase-2.6.5-core-reliability-remediation.md](phases/phase-2.6.5-core-reliability-remediation.md), [deferred-tasks.md](deferred-tasks.md), [../project-structure.md](../project-structure.md), [../tech-stack.md](../tech-stack.md) > **Authority (until Phase 2.5.5 and Phase 2.6 close):** This file is canonical for > live progress and execution order; the phase documents stay canonical for scope and acceptance. Ordering > differences between the two are deliberate. `current.md` may override recommended ordering — it may **not** > redefine a security boundary, a scope line, or a milestone acceptance criterion. -This page tracks what is active **right now** and the immediate next concrete actions. -The full phase plan and the global milestone spine are in [README.md](README.md). -**Phase 2.5 (CLI Consolidation) is complete** (milestone **M2.5-4**, PR #69, 2026-07-08) — its -breakdown, now historical, is in -[phases/phase-2.5-cli-consolidation.md](phases/phase-2.5-cli-consolidation.md). -A 2026-07-19 full-project code review then opened **Phase 2.5.5 (Hardening and Remediation)**, an -interstitial phase holding the review's verified findings against the pre-2.6 codebase; it runs -**concurrently with Phase 2.6, not before it** — see -[phases/phase-2.5.5-hardening-and-remediation.md](phases/phase-2.5.5-hardening-and-remediation.md) -and [Active now](#what-is-active-now). -**Phase 2.6 — Conversational Authoring and the First-Class CLI** is **in progress** (re-scoped -2026-07-08); its plan is in -[phases/phase-2.6-conversational-authoring.md](phases/phase-2.6-conversational-authoring.md). -Workstream **2.6.F (platform floor + the full-screen TUI renderer)** is **merged to `main`** -(PR #74, 2026-07-11), and so is **2.6.C** (the reseat transcript-carry + the `/cost` per-model breakdown, -PR #75, 2026-07-13) — see [Active now](#what-is-active-now). **2.6.Q** (dynamic model-catalog -enrichment) is **no longer blocked**: [ADR-0071](../decisions/0071-models-dev-as-the-model-metadata-source.md) -and [ADR-0072](../decisions/0072-model-metadata-in-the-db-behind-a-generated-offline-floor.md) are both -**Accepted**, resolving the six open maintainer questions the phase file had cited, and the P1–P5 -implementation steps are **merged to `main`** (PR #76, 2026-07-26). P6 and the `/settings` → `/models` -visibility toggle remain open; the bundled offline snapshot was **regenerated** in PR #79 after the D3 -price ruling. - -**The unified build order across both phases is in -[Execution order — Phase 2.5.5 + Phase 2.6](#execution-order--phase-255--phase-26-temporary) below** — a -temporary section, deleted when both phases close. Its baseline step is discharged (PRs #76 and #77 merged) -and Wave 0 is merged (PR #82), as is Wave 1 (PR #83). The interlude's `W2` (core reliability, six items) is -merged too (PR #85, 2026-08-28). - - -> **W7 merged to `main`, 2026-10-10 (PR #90, `46a6a701`):** all twelve steps, five genuine overflow records, -> production request ownership and consolidated host lifetimes have independent acceptance. -> All six W7 items are closed; Phase 2.6.5 stood at **47/51 at that merge**. The -> [closing register](phases/phase-2.6.5-core-reliability-remediation.md#w7-closing-register--2026-10-10) -> joins per-item causal controls, both final review rounds and every submitted systematic finding. -> PR #90 merged at 13:57:47 UTC. W8 was still open at that checkpoint; it subsequently merged in PR #92. Approved deferrals remain. -> No implementation, user decision, credential, billing or provider capture is pending for W7. - -> **Post-closure corrections accepted, 2026-10-10:** the [new correction register](phases/phase-2.6.5-core-reliability-remediation.md#post-closure-systematic-review--2026-10-10) -> joins every new report disposition. Core, CLI/DB and tooling/docs each have two NEW complete clean -> cumulative review rounds. Fresh unchanged-production coverage passes 447 files / 9,512 cases / -> eleven existing capability/key-gated LLM skips. Published-head checks are reported in PR #90. -> W7 has no implementation, provider, credential, billing or user-decision prerequisite left. -> The corrections merged with PR #90; W8 subsequently merged in PR #92. Existing approved limits remain. - -> **Publication tool corrections accepted, 2026-10-10:** [two NEW complete clean cumulative rounds](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-10T13-35-14-w7-publication-sonar-round-2-review.md) -> independently verify all four Sonar-label dispositions, preserved UTF-16 ordering and OS-owned Git -> dispatch with PATH-trap causal controls. Shipping production/tests remain unchanged. Final local CI -> and exact published-head GitHub/Sonar checks are reported in PR #90; the prior failed analysis stays recorded. -> W7 has no implementation, capture, credential, billing or user-decision prerequisite left. - -## W8 merged — Phase 2.6.5 complete - -**W8 merged to `main` on 2026-10-11 as [PR #92](https://github.com/HodeTech/Relavium/pull/92), -merge commit `ef0ffa54`. Phase 2.6.5 is complete with approved deferrals: 49/51 items closed.** -The [merge checkpoint](phases/phase-2.6.5-core-reliability-remediation.md#w8-merge-checkpoint--2026-10-11) -joins final publication evidence and all seven phase exit criteria. All seven W8 steps, -CR-81, CR-82 and ADR-0087 §1 are complete on `main`; no W8 implementation, review, -publication, provider, billing, credential or user-decision gate remains. - -1. **Consumption:** explicit run/session modes preserve iterated delivery from construction; - subscribe-only handles retain no primary buffer and reject iteration with a typed error. -2. **Conformance:** strict expected exchanges and the 24-case actual installed-SDK HTTP - matrix detect production-lowering regressions. -3. **Usage uncertainty:** complete/partial/missing observations, pending ACK before invocation, - once-only settlement and late receipts, sticky old-writer uncertainty and qualified - human/JSON totals preserve the distinction between unknown usage and actual zero. - -The [original closing register](phases/phase-2.6.5-core-reliability-remediation.md#w8-closing-register--2026-10-11) -preserves its source acceptance and two NEW complete whole-wave reviews. The -[maintainer correction register](phases/phase-2.6.5-core-reliability-remediation.md#w8-pr-92-systematic-review-corrections--2026-10-11) -joins all report dispositions and two NEW independent complete clean cumulative rounds per -correction increment. Final reviewed head `3d4491df` passed both seven-job PR/push CI -workflows and the SonarCloud quality gate. Published ordinary/enforced-coverage inventories -are 511 files / 11,026 passing tests / 12 established skips; four wire and 23 usage -mechanisms retain 427 permanent cases and 69 joined native children. Fresh full local -CI and bare/enforced coverage at `6139563e` remain separately attributed evidence. -No merge-commit execution, live-provider invoice or unsupported-platform result is inferred. -Two nonblocking Sonar complexity suggestions remain visible. Detailed review records stay -local and ignored; historical citations remain commit-pinned. - -**What remains:** CR-70 (cross-turn tool-history carrying) and CR-93 (tenant-scoped mutable -state) are approved deferrals. The 49/51 count includes CR-95's short-term fail-closed fix; -its long continuation and existing exact-capacity/scoped residuals remain in -[deferred-tasks.md](deferred-tasks.md), with their existing severity, trigger and narrowed -claim. ADR-0087 §1 is outside the denominator. Phase closure does not claim 51 fully -implemented items or an empty debt backlog. Phase 2.5.5 and Phase 2.6 remain in progress. -Earlier dated pending checkpoints in this page and the phase register are historical. - - - - - -## Wave 2 merged — Shut the doors - -**Wave 2 merged to `main` on 2026-10-15 as [PR #94](https://github.com/HodeTech/Relavium/pull/94), -merge commit `ab08209e`. All eight steps and the maintainer corrections are complete.** -The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) -joins the forty item dispositions, four new security sittings plus W4 revalidation, -original whole-wave acceptance and all subsequent correction scopes. Existing incoming -navigation anchors are retained; dated implementation checkpoints remain historical. - -Completed work covers invocation-authorized filesystem boundaries and scratch-policy -composition; bounded secret stdin, early admission and private bridge diagnostics; -faithful-snapshot warnings, structural session privacy and export completion; verified -bounded media/CAS/native lifecycles; and the interim provider-header gate. Maintainer -corrections include runner-level error rejection, valid Opus/Vorbis publication, -FLAC structural consistency, generated-result streaming and finite pin-error provenance. - -Final reviewed correction source `4cdfe011` has two NEW independent complete clean -cumulative rounds covering all 67 correction paths / 188 hunks / 24 complete new files. -These scoped rounds do not claim a new review of every original PR path. Full local -CI, ordinary/enforced coverage and all nine jobs in each PR/push CI workflow pass; -SonarCloud and selected native Windows/Node-floor controls pass. Local coverage records -588 files / 12,983 passing tests / 20 explicit skips; CI records 12,982 / 21 with its -established extra CI-only skip. Complete physical controls are independently inspected. -The merge tree matches the reviewed source; no new merge-commit execution is inferred. - -**Satisfied on `main`:** Phase 2.5.5 exit criteria 1–3 and sub-stream D acceptance. -**Still open:** M2.5.5-2's I half, whole Phase 2.5.5/2.6, M/N production lifecycle, -future live headers and existing approved residuals. Wave 2 has no remaining implementation, -review, publication, ADR approval, provider capture, credential, billing or user-decision gate. -The [deferred register](deferred-tasks.md) retains its existing limits; closure adds no deferral. - -## Wave 3 Steps 1–8 merged — PR #95 - -**Wave 3 Steps 1–8 merged to `main` on 2026-10-18 as [PR #95](https://github.com/HodeTech/Relavium/pull/95), -merge commit `6d2a7372` (22:44 UTC). Wave 3 itself is not complete: Steps 9–19 had not started at -that merge (Steps 9a.1, 9a.2a and 9a.2b are now accepted on `development`, the native-identity increment NI and Steps 9b.1 and 9b.2 are accepted too, and Steps 9c–19 have not started) and whole-wave acceptance -remains Step 19.** The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) -joins the completed scope, every review attribution, the independent review's dispositions and the -publication state observed at merge. It was the maintainer-authorized, scoped publication checkpoint. - -On `main` now: close handling, safe source labels and exhaustiveness guards (Step 1); shared admission and -diagnostics (2); secret provenance at public sinks, ADR-0113 (3); the typed error/envelope substrate (4); tool and -engine tails (5); the pure process/discovery substrate, ADR-0110 (6); the lazy runtime, finite compiled graph and -exact-tarball consumers, ADR-0111 (7); and creation-owned native process integration (8). - -Three points were recorded rather than papered over; all three are now resolved. The merged head `a3e164b5` carried -corrections from a further independent review (a secret-scrubbing production rule, two behaviour-neutral import moves, -tests and causal-control pins) with **author verification only** at merge; a post-merge independent review — four -rounds and a security sitting, with fixes in follow-up commits — accepted them, as the -[review record](phases/phase-2.5.5-hardening-and-remediation.md#independent-review-of-a3e164b5-and-its-follow-up-corrections--2026-10-19) shows. CI had not finished when the merge -happened (the required `lint · typecheck · test` job was still running); the three runs later completed `success`, -15 of 15 jobs each, as the [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#publication-state-observed-at-merge) -records. And SonarCloud's quality gate, which had failed on `main` since at least PR #90 because its dispositions lived -only on PR branches, was cleared on 2026-10-19 (zero open vulnerabilities, gate OK). Open findings remain, including two -security residuals that predate the follow-up — an `api-key:Bearer ` token leak and unmasked userinfo with a raw -`?`/`#` — in [deferred-tasks.md](deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18). - -## Next work — Wave 3 — Clear the ground - -Wave 3 implementation continues on `development`: Steps 1–8 are merged. A pre-implementation document/source review -of Steps 9–19 and the maintainer's rulings are recorded in the -[2026-10-19 rulings](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) -(no new ADR; 29 recorded increments). **The PR95 increment is closed on `development`** — the PR #95 -scrubber residuals, including the two security residuals — under the maintainer's 2026-10-21 masker acceptance -bar and round cap, as its -[implementation checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20) records: rounds 1–10 were NONCLEAN; the high and -medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the -maintainer counted as high on 2026-10-22, is corrected (in the summary, by the author alone). Its other findings -are recorded in -[deferred-tasks.md](deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is independently accepted on `development` at `cf97b7f1` (2026-10-24): its round 13 reported no blocker, high or medium finding and its security sitting was clean, and the round's low findings are recorded in deferred-tasks.md ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a1-acceptance--2026-10-24)); Step 9a.2 is split into 9a.2a and 9a.2b, and Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24) after three rounds, the third with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2a-acceptance--2026-10-24)); Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24) after two rounds, the second with no blocker, high or medium finding and no regression ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2b-acceptance--2026-10-24)), and the native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25) after one round with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)), its Windows clause open until three green advisory Windows runs; Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b-preflight-decisions--2026-10-25)), and Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is independently accepted on `development` at `147c2788` ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b1-acceptance--2026-10-25)), and Step 9b.2 (the release binding receipt v1, verified ciphertext reuse, the absent production recipient loader and scope separation) is independently accepted on `development` at `563402e6` ([acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9b2-acceptance--2026-10-26)), with Step 9c next; the substrate's -9a.1–9d increments carry a dedicated archive/crypto security sitting. Steps 10–19 follow the -[increment table](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) and end with -whole-wave acceptance at Step 19. `development` is pushed after each accepted increment; PR #95 no longer bounds the -review scope of new commits. See the [Wave 3 queue](#wave-3--clear-the-ground). - -## Historical W7 implementation checkpoints - -Dated W7 checkpoints throughout this page retain their original pending states. The closing checkpoint above -and its later post-closure and merge checkpoints control current W7 status; historical pending/draft -wording describes the checkpoint when it was written. - -> **Prior complete composition review:** [host-closure round 5](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-10T00-38-24-w7-host-closure-round-5-review.md) -> finds no new confirmed actionable issue across all 126 paths / 364 hunks. This is the first -> clean complete cumulative round; one further fresh complete clean round remains required. -> Step 8 and final Step 12 remain open, with no user/provider blocker. -> -> **Complete composition review:** [host-closure round 4](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-10T00-22-00-w7-host-closure-round-4-review.md) -> confirms public stream exit can skip underlying cleanup on retirement failure. Parent adds -> twelve paired permanent cases and guarantees cleanup while preserving the original throw. -> Two fresh complete clean rounds, Step 8 and final Step 12 remain required; no user input is pending. -> -> **Complete composition review:** [host-closure round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T23-56-41-w7-host-closure-round-3-review.md) -> confirms two retirement-boundary Highs: mutable returned output/usage and failed cleanup -> suppressing accounting/acknowledgement. Parent corrections add thirty-two permanent cases. -> Fresh complete review remains required; Step 8/12 remain open, with no user/provider blocker. -> -> **Complete composition review:** [host-closure round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T23-34-00-w7-host-closure-round-2-review.md) -> confirms due-budget departure starvation and provider entry after synchronous cancellation. -> Parent corrections add nineteen permanent cases and fix adjacent canonical-documentation drift. -> Fresh complete review is required; Step 8/12 remain open. No user/provider blocker exists. -> -> **Complete composition review:** [host-closure round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T23-05-00-w7-host-closure-round-1-review.md) -> confirms three lifecycle High findings (post-pause receipt cursor, rejected activation joining, -> post-unmount Ink visibility) and a required shipping-registry acceptance gap. Parent corrections -> add eighteen permanent cases and strengthen native output assertions. Fresh complete review of -> the corrected composition remains required before acceptance; Step 8/12 are still open. - -> **W7 live status, 2026-10-09:** all five immutable provider records and Step 7's four-round -> acceptance remain complete. [Two fresh adapter/chain round-3 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-08T23-47-31-w7-owned-adapter-chain-round-3-review.md) -> accept all 24 cumulative paths/78 hunks through `0c3c185b`; exact-head required CI, coverage, -> Node 22, Windows, peers and Sonar pass. Core measured-round reuse is implemented at `99e78e55`; -> [both independent round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T00-16-47-w7-owned-core-round-1-review.md) -> clear all 13 paths/47 hunks. [Two fresh round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T01-00-00-w7-owned-core-round-2-review.md) -> accept all 19 cumulative paths/53 hunks through `f1281214`; exact-head CI passes. ADR-0103 -> append/receipt integration is implemented at `a35739a4`; -> [both complete round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T01-27-00-w7-append-receipt-round-1-review.md) clear all 17 paths/57 hunks. -> [Two fresh cumulative round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T02-00-00-w7-append-receipt-round-2-review.md) accept all 23 paths/63 hunks -> through `01115050`. Raw provider/iterator/media and native child lifetime integration is -> implemented at `fde3ca0c`; [both complete round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T02-36-10-w7-raw-producer-round-1-review.md) -> verify a public-return High, corrected with two regressions (28 permanent cases total). -> [Two fresh cumulative round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T03-25-00-w7-raw-producer-round-2-review.md) accept all 20 paths/41 hunks -> through `1172d247` after complete report/artifact/freeze audits; nested Corepack reviewer -> violations get no proof credit. The HTTP descendant increment now has 23 new permanent -> cases, 16 compiled negative controls and a passing cache-qualified root CI; -> [both complete round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T04-16-24-w7-http-descendants-round-1-review.md) -> clear all 22 paths/38 hunks through `db02b83c` after Parent's complete artifact/freeze audits. -> [Two fresh cumulative round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T04-55-00-w7-http-descendants-round-2-review.md) -> accept all 24 paths/40 hunks through `6e620556` after Parent's complete evidence and 645,606-entry freeze audits. -> Invocation-local provider/raw-poll integration adds 57 permanent cases; [complete round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T05-43-00-w7-provider-invocation-round-1-review.md) -> verify a method-receiver defect and separately strengthen a weak Promise test. -> [Both complete round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T06-33-00-w7-provider-invocation-round-2-review.md) verify one getter-acquisition defect. -> Its correction passes root CI (404 files / 9,030 cases / 12 existing skips); -> [both NEW complete cumulative round-3 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T07-06-28-w7-provider-invocation-round-3-review.md) -> and Parent's complete artifact/645,743-entry freeze audits accept all 28 paths/67 hunks through -> `f9a8ff37`, with no new confirmed finding. Exact-head required CI, coverage and Sonar pass. -> MCP transport/handler descendants now have 77 new permanent cases. Rounds 1–3 corrected -> queued-reservation, deadline, disposer, coverage, empty-start stdio and callback failures. -> [Complete round-4 reviews and Parent evidence/freeze audits](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T12-58-01-w7-mcp-lifetime-round-4-review.md) verify -> completed control-lane reference retention. Correction `37220a95` releases only fully -> acknowledged lanes and passes actual root CI; seven new causal regressions and fresh enforced -> private coverage pass (412 files / 9,107 cases / 12 skips; MCP branches 90.68%). -> [Two NEW complete cumulative round-5 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T13-37-00-w7-mcp-lifetime-round-5-review.md) -> and Parent's complete source/artifact audits accept all 38 paths/70 hunks through `99c46bbd`, -> with no new confirmed finding and 268 independently passing cases per reviewer. The full -> shared-tree audit has zero file/link changes and one qualified `.git` directory mtime change. -> Fresh startup registers its complete continuation. [Two complete first-round reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T14-28-09-w7-startup-lifetime-round-1-review.md) -> and Parent audits verify reentrant clock/timer cancellation; three permanent regressions -> cover the correction. [Two NEW complete cumulative round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T14-55-12-w7-startup-lifetime-round-2-review.md) -> and Parent's exact-source/physical-artifact/647,041-entry freeze audits accept all ten -> paths/sixteen hunks through `5d44b13b`, with zero new confirmed findings and 2,850 core cases per reviewer. -> [Both complete scheduler readiness round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T15-44-15-w7-scheduler-readiness-round-1-review.md) -> verify a secondary readiness-diagnostic failure. Its correction has twelve causal readiness -> regressions and 2,862 passing private core cases. [Two NEW complete cumulative round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T16-14-29-w7-scheduler-readiness-round-2-review.md) -> and Parent's source/artifact/647,245-entry unchanged freeze audit accept all nine paths/ten -> hunks through `95f3923a`, with zero new readiness findings. Both inherited dispatch Highs -> (post-start-append cancellation admission and unsafe detached failure formatting) are -> corrected in `1918690d`, with six new permanent cases and passing exact-input root CI. -> [Both complete dispatch-correction round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T17-28-06-w7-dispatch-corrections-round-1-review.md) -> and Parent's artifact/647,355-entry unchanged freeze audits accept the eight-path/ -> seventeen-hunk scope with zero confirmed findings. [Both complete cumulative second-round reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T17-58-41-w7-dispatch-corrections-round-2-review.md) -> verify an inherited retry-start cancellation High. Its correction and a subsequent -> Parent-discovered synchronous attempt-entry correction add ten permanent cases; -> final-source causal controls pass. NEW complete cumulative reviews are required -> before scoped closure; complete scheduler/dispatch roots stay open. -> [Complete cumulative round-3 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T18-37-30-w7-dispatch-corrections-round-3-review.md) verify a retry/deadline duplicate-terminal High. -> Parent confirms and corrects it, with a new permanent causal regression; fresh cumulative review remains required. -> [Complete cumulative round-4 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T19-10-39-w7-dispatch-corrections-round-4-review.md) verify two paths of one attempt-correlation High. -> Parent corrects explicit failure forwarding and first-start reset with five permanent cases; NEW cumulative review is required. -> [Complete cumulative round-5 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T19-41-03-w7-dispatch-corrections-round-5-review.md) verify a pre-start readiness attempt-correlation High. -> Parent corrects its latest-entered-attempt forwarding with two paired permanent cases; NEW cumulative review is required. -> [Complete cumulative round 6](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T20-08-19-w7-dispatch-corrections-round-6-review.md) finds no new confirmed issue. -> [The NEW complete round 7](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T20-29-02-w7-dispatch-corrections-round-7-review.md) verifies a differing-order grace High: -> an unstarted claim can receive a failure. Parent corrects explicit claim/start tracking, -> with three permanent causal cases; NEW complete cumulative reviews remain required. -> A broader private actor prototype exposed CLI closure/lease-handoff failures in root CI. -> Complete scheduler/dispatch/resume registration must land with shipping host departure; -> original failures and assertions are retained, and that integration stays required in W7. -> Complete MCP startup/all-actor retirement, custom-provider descendants and remaining host departure work -> and Step 8 atomic compaction/recovery remain open. The ownership/lifecycle -> Highs, Step 12, final whole-wave CI/coverage/Sonar and all six W7 register items remain open. -> Draft PR #90 is unmerged; no further paid call, credential or ADR approval is needed. -> Earlier progress statements below describe their publication checkpoints. - -> **W7 implementation, 2026-10-02:** the documentation review is closed. The maintainer approved -> [ADR-0099](../decisions/0099-compaction-has-an-idle-budget-outcome-and-an-unknown-window-policy.md), -> [ADR-0100](../decisions/0100-budget-authorization-is-durable-state-with-a-replay-barrier.md) and the -> [twelve-step W7 plan](phases/phase-2.6.5-core-reliability-remediation.md#w7-pre-implementation-review-and-proposed-execution-plan--2026-10-02). -> Work remains on `development`; each step closes with two fresh independent Codex review rounds and verified -> fixes. Step 1's implementation and first review corrections pass `pnpm run ci` and `pnpm coverage`; -> [round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T03-04-05-w7-step-1-round-1-review.md) verified seven findings; -> [round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T03-22-32-w7-step-1-round-2-review.md) verified a cleanup race in its fix. -> [Round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T03-35-41-w7-step-1-round-3-review.md) verified a passing/starting race; -> cleanup now retains shared parents and deletes only owned probes. All corrections pass the gates; -> [Round 4](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T03-43-31-w7-step-1-round-4-review.md) cleared probe ownership and found -> a capture pathname-cleanup race. The runner now closes its descriptor without deleting a failed -> destination. [Round 5](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T03-50-35-w7-step-1-round-5-review.md) cleared the -> complete step and final corrections; step 1 is closed. Step 2's structural store boundary and -> durable effect-turn allocator pass the required checks and root coverage (289 suites, 6,367 passing -> tests, 11 skipped); full CI passed at `1927daa1`. -> [Round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T04-29-12-w7-step-2-round-1-review.md) verified nested-transaction -> key reuse and unsafe session media metadata. Both are corrected; checks and coverage pass -> (289 suites, 6,374 passing tests, 11 skipped), with full CI passing at `c96eb9b5`. -> [Round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T04-41-54-w7-step-2-round-2-review.md) cleared the complete step -> and both fixes, including real lock contention and process-crash checks. Step 2 is closed; -> step 3 implementation now connects structural completed turns, atomic persistence, empty-final -> resume/export and per-call effect ids across the CLI hosts. Required checks and coverage pass; -> [Round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T05-34-44-w7-step-3-round-1-review.md) verified a deep-JSON -> metadata failure and a lost policy-denial classification. Both corrections pass the required -> checks and coverage (291 suites, 6,406 passing tests, 11 skipped); full CI passes at `4045272c`. -> [Round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T05-48-54-w7-step-3-round-2-review.md) verified a cached-command -> durability bypass and a missing tool name on recovered filesystem denial. Both corrections pass -> checks and coverage (291 suites, 6,410 passing tests, 11 skipped); full CI passes at `e95a6010`. -> [Round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T06-10-38-w7-step-3-round-3-review.md) independently verified an -> unjournaled-tool durability bypass and a stale export introduction. Dispatch admission now checks -> the live latch after approval/preparation for every actual tool call and releases a proven -> non-dispatch claim. Both corrections pass checks and coverage (291 suites, 6,415 passing tests, -> 11 skipped); full CI passes at `dd50738c`. -> [Round 4](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T06-34-39-w7-step-3-round-4-review.md) cleared dispatch admission -> and verified loss of nontrailing pre-W7 empty-final user text from resume/reseat and export prompts. -> Legacy context and boundary slots now preserve the old read behaviour without fabricating terminals -> or completed counts. Required checks and coverage pass (291 suites, 6,422 passing tests, -> 11 skipped); full CI passes at `6d5b3f0b`. -> [Round 5](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T06-48-45-w7-step-3-round-5-review.md) cleared the complete step -> and all earlier corrections, including independent mixed-history projection, YAML, real SQLite -> boundary and process-crash probes. **Step 3 is closed; step 4 proceeds automatically** with -> session-result privacy, full-history disclosure snapshots and exact committed-row retention. -> Step 4's [first acceptance round](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T08-05-02-w7-step-4-round-1-review.md) -> independently reproduced pre-mount Ink resume/reseat deletion. The verified correction separates -> opening from committed-mount activation and carries driver activity through disclosure and teardown; -> [Round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T08-48-12-w7-step-4-round-2-review.md) reproduced activation before passive Ink input setup -> and before disclosure was drawn. Activation now awaits usable setup and disclosure render acknowledgement, -> observes renderer exit/error and keeps input gated; actual Ink and Home tests verify the display boundary. -> Required checks and coverage pass (297 suites, 6,487 passing tests, 11 skipped); a third fresh -> corrective acceptance round was required before step 4 closes. -> [Round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T09-24-27-w7-step-4-round-3-review.md) verified incomplete-call attribution and -> flush acknowledgement of unwritable output. Completion now uses the shared structural projector; -> rendered disclosure checks the actual Ink output and its lifetime. Required checks, build and coverage -> pass (297 suites, 6,497 passing tests, 11 skipped); a fourth fresh acceptance round is required. -> [Round 4](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T10-04-04-w7-step-4-round-4-review.md) verified headless stderr enqueueing mistaken for delivery -> and inline disclosure lost during Ctrl-Z suspension. Native stderr callbacks now acknowledge delivery; -> notice publication and render acknowledgement share terminal ownership with suspension. Nine focused suites -> pass 217 tests, including delayed native delivery and independent real STOP/CONT controls. -> Required workspace checks, build and coverage pass (298 suites, 6,514 tests, 11 skipped). Full CI -> passed at `b9d52d76`; round 5 required two more corrections. -> [Round 5](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T10-36-22-w7-step-4-round-5-review.md) independently reproduced blocked native stderr -> holding Ctrl-C teardown and late Home acknowledgement deleting evidence during canceled-session MCP close. -> Headless activation now races an owned interrupt; Home activation requires a running session outside teardown. -> Eight focused suites pass 310 tests; native pipe/FIFO before/after controls and late callbacks after actual -> database close pass. Workspace checks, build and coverage pass (298 suites, 6,520 tests, 11 skipped). -> Full CI passed at `e753beb7`. [Round 6](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T10-57-58-w7-step-4-round-6-review.md) accepted the complete -> step and every correction with no new material findings. Parent native/privacy controls pass; compiled-entry -> process-exit delay under queued stdio also reproduces on the exact pre-step baseline and is explicitly qualified. -> **Step 4 is closed; step 5 proceeds automatically.** All six W7 register items remain open. -> Step 5's memory implementation and canonical docs are checked: -> one pure request projection, frozen policy, completed-turn spans and engine/CLI history refusals; -> actual request, workflow-node and SQLite resume/reseat controls pass. Workspace checks, build and coverage -> pass (299 suites, 6,567 passing tests, 11 skipped). Later measured pre-send/recovery and revised compaction -> outcomes remain staged. Clean-tree full CI passed at `ecdcc395`. -> [Step 5 round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T12-00-17-w7-step-5-round-1-review.md) accepted the implementation: -> both reviewers passed 347 focused tests; parent-rerun controls passed 390 state/history cases and 86 -> actual CLI/SQLite cases. Matched source-head operational controls pass; exact pre-step engine controls -> fail as expected. Adjacent working-context/durable-history wording is corrected. -> [Step 5 round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T12-21-33-w7-step-5-round-2-review.md) accepted the complete step: -> both fresh reviewers passed 347 tests; the parent reproduced 1,536 state/history and 86 CLI/Home/SQLite -> cases, plus exact-source positive/negative controls. **Steps 1–5 are closed; all six W7 items remain open.** -> Step 6's independently reviewed [Accepted ADR-0101](../decisions/0101-configured-output-estimates-apply-only-when-the-wire-is-uncapped.md) -> resolves a newly identified configured-estimate/native-cap precedence gap. The maintainer approved the -> complete revised decision on 2026-10-02; dated forward notes are landed and step 6 proceeds; -> [ADR revision round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T15-09-58-w7-adr-0101-revision-round-1-review.md) records the maintainer's -> verified transport/pricing findings and the corrected measurement-to-attempt handoff. -> [Fresh draft round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T15-18-01-w7-adr-0101-revision-round-2-review.md) accepted the complete -> corrected proposal with no actionable findings; full CI passed at `2b628e1a`. The acceptance and runtime -> obligations remain distinct: runtime acceptance is now complete. -> [Step 6 round 8](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T22-44-01-w7-step-6-round-8-review.md) accepts the complete step and seven corrections. -> Both fresh reviewers reported no verified new findings; parent replays pass 120 contract and 54 -> financial controls. **Steps 1–6 are closed; all six W7 register items remain open.** -> Step 9's quote/debit foundation is implemented, with [candidate preflight and parent causal controls](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T23-40-16-w7-step-9-foundation-preflight-review.md). -> Full CI/coverage pass after the round 3 correction (315 test files, 7,152 passing tests, 11 existing skips). -> [Round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T00-25-08-w7-step-9-round-1-review.md) verified and corrected two caller settlement findings; [Round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T01-10-45-w7-step-9-round-2-review.md) verified and corrected the async consumer and cancellation teardown; [Round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T01-57-14-w7-step-9-round-3-review.md) verified and corrected park/poll host admission loss and the reentrant durable-total race. -> [Fresh round 4](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T02-40-53-w7-step-9-round-4-review.md) accepts the complete foundation with no new findings; both sealed inventories and 70 parent-rerun fresh cases are verified. Its corrected native-loader evidence and prior harness isolation exception are recorded explicitly. -> **Step 9 is closed; approved Step 10 proceeds automatically** with durable workflow activation and replay. -> [Step 10 round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T08-35-49-w7-step-10-round-1-review.md) reproduced and corrected -> passive-resume lease expiry and unbound predecessor CJS/peer/optional edges. Exact-fence renewal and -> an owned, versioned dependency closure pass causal controls, full CI and coverage -> (320 suites, 7,318 passing tests, 11 existing skips), including seventy actual-runner cases, -> twelve real predecessor refusals, two legacy controls and seven permanent graph/runtime controls. -> [Step 10 round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T11-47-48-w7-step-10-round-2-review.md) verified and corrected -> terminal conservative-money ordering, historical budget input compatibility, companion amount symmetry -> and pinned runtime edges. Full CI/coverage pass (320 suites, 7,340 passing tests, 11 existing skips); -> 78 native runner cases, thirteen graph controls and actual historical-input replay pass causal checks. -> [Step 10 round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T12-39-00-w7-step-10-round-3-review.md) verifies and corrects -> authority-witnessed optional-amount duplicate compatibility, preserving strict money conflicts, -> genuine output and unrelated pending work. Eighteen permanent regressions and 300 native selected -> cases pass. Full CI/coverage pass (320 suites, 7,358 tests, 11 existing skips); fresh full-Step-10 -> round 4 precedes closure. All six register items remain open. -> [Step 10 round 4](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T13-33-00-w7-step-10-round-4-review.md) corrects lease cleanup rejection replacing -> primary typed refusals or an already-terminal no-op. Eighteen native regressions cover all six -> guards and successor takeover; full CI/coverage pass (320 suites, 7,376 tests, 11 existing skips). -> Fresh whole-Step-10 round 5 precedes closure; no residual is deferred. -> [Step 10 round 5](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T14-31-39-w7-step-10-round-5-review.md) corrects inherited parallel gate publication handing off -> and then reclaiming a waiting-only lease. Ten permanent regressions cover held pause ACKs, -> actual SQLite fresh-owner resume and immediate human-decision media preparation. Async final-ACK -> limits are explicit; no new clock/timeout policy is chosen. Full CI/coverage pass (320 suites, -> 7,386 tests, 11 existing skips). Fresh whole-Step-10 round 6 precedes closure. -> [Step 10 round 6](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T15-42-00-w7-step-10-round-6-review.md) corrects inherited ordinary -> decision media completion after cancellation, deadline or fenced closure. Twelve permanent -> Core/actual SQLite-CAS regressions prevent late subscriber success and renewed run references; -> guard removal/restoration confirms the failure. Full CI/coverage pass (321 suites, 7,398 tests, -> 11 existing skips). Fresh full-Step-10 round 7 precedes closure. -> [Step 10 round 7](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T16-54-28-w7-step-10-round-7-review.md) corrects inherited media references written before a stale -> event fence is refused. Produced handles now follow append acknowledgement; owned uncertain-terminal -> media remains retained for the outbox. Seven permanent successor/terminal controls and independently -> verified sealed evidence pass, with full CI and sequential coverage (323 files, 7,405 tests, -> 11 existing skips). Fresh full-Step-10 round 8 precedes closure. -> [Step 10 round 8](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T17-50-23-w7-step-10-round-8-review.md) accepts the full 74-file step with no new verified findings. -> Both fresh reviewers pass 857 permanent tests and separate five/six fresh probes; complete reports and -> every sealed inventory entry are parent-verified. **Step 10 is closed; approved Step 11 proceeds automatically** -> with the safe CLI surface, permanent compiled-process smoke coverage and fresh independent reviews. -> Step 11 implementation now passes full CI and sequential coverage (327 files, 7,506 tests, 11 existing skips), -> including permanent compiled CLI refusal/status/abort/MCP-interruption smokes and raw-triple display regressions. -> [Step 11 round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T19-39-26-w7-step-11-round-1-review.md) independently verifies native approval-write cancellation, -> the no-MCP signal guard and stale identified-companion display cleanup. The parent fixes all three, -> adds passive Core cancellation/fence controls and corrects the ownership wording. -> Full CI and sequential coverage pass after the corrections (327 files, 7,519 passing tests, -> 11 existing skips; global line/branch/function 95.88%/92.63%/96.73%). -> [Step 11 round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T21-01-16-w7-step-11-round-2-review.md) verifies two further High findings: -> a successful native busy wait can conceal OS SIGINT until after egress, and terminal write uncertainty -> was misreported as takeover. The CLI host yields across a poll turn before decision acknowledgement; -> terminal uncertainty retains exit 5, while actual successor fencing still refuses without a false -> claim that an earlier approval is absent. Seven new permanent controls pass; both complete sealed -> reports and every inventory entry are parent-verified. Full CI and sequential coverage pass -> (327 files, 7,526 tests, 11 existing skips; 95.88%/92.63%/96.73%). Fresh full-Step-11 round 3 follows -> the correction commits; the step remains open. -> [Fresh Step 11 round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T01-17-23-w7-step-11-round-3-review.md) is clean in both -> complete independent reviews: 452 permanent tests each plus separately reported fresh native, -> passive and compiled-process controls. Both reports and every sealed inventory entry are verified. -> Interrupted earlier round-3 runs contribute no acceptance. **Step 11 is closed.** -> [Independent Step 12 document preflight](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T01-04-15-w7-step-12-document-preflight-review.md) -> verifies and corrects usage/pricing and fixture-coverage prose; connected at-rest/event claims, -> guides and moved follow-up status are reconciled. This independent maintenance proceeds through -> repository checks and fresh post-commit review; Step 12's whole-wave acceptance remains open. -> [Committed maintenance round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T07-00-17-w7-step-12-maintenance-round-1-review.md) -> independently verifies one Medium reservation-existence qualification and one Low inherited -> event-table guarantee. Both are parent-verified and corrected in four text/comment locations. -> Complete reports and all 2,687 sealed entries are verified; 81 fresh reviewer DB tests and -> a real-writer reference-corruption/restoration control pass. Fresh complete maintenance -> round 2 follows the correction commits; whole Step 12 remains open. -> [Committed maintenance round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T07-25-20-w7-step-12-maintenance-round-2-review.md) is clean -> across all 26 paths/37 hunks in both fresh reviews. Both complete reports and all 2,574 sealed -> entries/source pins are verified; fresh 81-test DB/strict checks and real-writer causal controls pass. -> **The independent maintenance subset is closed; whole Step 12 and all six W7 items remain open.** -> [Draft PR #90](https://github.com/HodeTech/Relavium/pull/90) is open on `development`. -> Required Linux CI revealed an actual-Ink test-context mismatch. Further controlled checks found -> undisplayed disclosure under the existing CI opt-outs and Home's inline reseat Static cursor; -> corrections pass two fresh independent review rounds. Prior scoped acceptance does not waive other findings. -> All five required actual overflow records (Anthropic input rejection and native context stop, -> OpenAI, DeepSeek and Gemini) are now preserved in the -> [fixture index](../../packages/llm/src/conformance/fixtures/overflow/README.md). -> The separate Anthropic native context-stop response was captured on 2026-10-08; -> the evidence prerequisite is satisfied. Step 7 classification, Step 8 and whole-wave acceptance remain open. -> Step 1 repaired collection without deleting private artifacts and provides the -> [maintainer-run live overflow capture procedure](../runbooks/capture-provider-overflow.md). -> Classification must replay the preserved live fixtures. The six W7 items stay open. - -> **PR 90 systematic follow-up — 2026-10-04.** Two maintainer reports against `95137c34` register 35 -> deduplicated findings, with [qualified independent triage](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T10-35-00-w7-systematic-review-intake-review.md). -> Step 6's request-ownership guarantee is reopened; adjacent session/CLI, replay-tool and documentation -> obligations require correction and fresh acceptance. The [first committed rendering review](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T10-32-37-w7-rendering-round-1-review.md) -> reproduces notice loss after a supported chat resizes and CI route/ownership drift. Its corrections -> are accepted by [fresh round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T12-00-00-w7-rendering-round-2-review.md). -> The rendering corrective group is closed. Previous step closure records remain historical evidence, -> not current whole-wave acceptance. All ten GitHub workflow checks pass at `d4e4fa9c`; SonarCloud and remaining findings stay -> separately under examination. No finding is deferred at intake. Steps 7–8, whole Step 12 and all six -> W7 register items remain open. - -> **Request-ownership decision preflight — 2026-10-04.** [Proposed ADR-0102](../decisions/0102-a-measured-request-owns-its-inert-data-through-egress.md) -> would close the measured-to-sent alias gap through one owned inert construction and existing candidate -> cap authority. It deliberately narrows non-cap opaque inputs and requires maintainer approval before -> dependent implementation. [Draft round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T12-21-41-w7-adr-0102-round-1-review.md) -> corrects SDK shared-reference parity and incoming measured-plan ordering. [Fresh complete round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T13-02-54-w7-adr-0102-round-2-review.md) -> receives no new material proposal findings; maintainer approval remains required before dependent -> implementation. Independent corrective work continues. The ownership HIGH remains open. - -> **Systematic group 2a, round 1 — 2026-10-04.** [Two complete independent reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T13-47-56-w7-systematic-group-2a-round-1-review.md) -> verify the cap/structural repairs but reproduce a new successful-tool observer regression: -> real session usage is lost and the hard turn cap can be bypassed. Corrections retain classified -> engagement/usage and fix the normalizer comment. Forced root checks and build pass; **fresh -> complete round 2 precedes group closure**. Ownership, remaining groups and W7 acceptance stay open. - -> **Systematic group 2a, round 2 — 2026-10-04.** [Both complete reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T14-41-03-w7-systematic-group-2a-round-2-review.md) -> verify the successful observer correction but reproduce an inherited failed-tool observer High. -> `41a3ef9a` retains real usage/engagement for invalid-argument and read-failure notifications, -> including EA7 abort, without broadening registry recovery. Eight new regressions fail before -> repair; forced root checks/build pass afterward. **Fresh complete round 3 precedes closure.** -> Ownership, other groups and whole W7 acceptance remain open. - -## Execution order — Phase 2.5.5 + Phase 2.6 (temporary) - -> **Temporary section — delete it when both phases close.** It owns **ordering only**. Every item's -> definition, size, file list and finding ids live in its phase file — -> [phase-2.5.5](phases/phase-2.5.5-hardening-and-remediation.md) (176 work items, sub-streams A–I) and -> [phase-2.6](phases/phase-2.6-conversational-authoring.md) (15 open workstreams) — and nothing below -> restates them. Items are cited as ` · ()`, which -> greps directly against the phase file. -> -> Derived 2026-07-26 from a full pass over both phase files, their own sequencing sections, and the live -> tree. Where it departs from a phase file's own prose, see [Departures](#departures-from-the-phase-files-own-sequencing). - -### Ordering root — one baseline ✅ **discharged 2026-07-26** - -The plan opened on a divergence neither phase file modelled: Phase 2.5.5's whole backlog is written against -`f88b0e8`, which existed only on `development` behind PR #76, so every `file:line` citation in it resolved -against a tree nobody was branching from. **PR #76 and #77 are both merged** — `f88b0e8` is on `origin/main`, -`origin/development` and `origin/main` held identical trees with no open PRs as of 2026-07-26. Every -citation in this plan now resolves. (PR #79 and PR #80 have opened since.) - -One consequence survives and is worth keeping until the phases close: - -> **Branch from `origin/development`, never from a local `main`.** A long-lived `development` merged into -> `main` by merge commit leaves a stale local `main` behind after every PR, silently and without conflict. -> Run `git fetch` and cut from `origin/development`. - -Also settled while discharging this: `main` now carries branch protection (PR required, `lint · typecheck · -test` required, force-push and deletion blocked) and a `v*` tag ruleset restricting tag creation to the -maintain/admin roles — closing the Phase-0 obligation that made `ci.yml`'s own "REQUIRED status check" -comment untrue. `release.yml` asserts its ref is reachable from `main` before `pack`, so **2.5.5.H's `G26` -is closed ahead of Wave 0**. - -### The three axes this order is built on - -1. **Irreversibility** — a defect whose damage a later fix cannot undo jumps every queue. That is Wave 1: - a credential already written into a durable `--json`-visible row, a billed turn lost after the reply - rendered, model-controlled ANSI inside the consent card, and the states where the cost cap is silently off. -2. **File ownership** — whoever edits a file next inherits everyone else's rebase. Every god-file - decomposition and naming/convention ruling lands *before* the 2.6 workstream that piles onto it, and - eleven items are pulled across the phase line because they edit the exact function another item restructures. -3. **Certification** — a wave that discharges a phase obligation certifies it while the evidence is fresh, - which is why 2.5.5's exit criteria 1–3 close in Wave 2 rather than at the end. - -Waves are readable single-branch sequences. Concurrency is expressed as *"these queues are file-disjoint, -any order"* — never as headcount. +This page tracks what is active **right now** and the order of the next work. The full phase plan and the global +milestone spine are in [README.md](README.md). Finished work keeps its record in the phase registers; this page +links to those records instead of restating them ([History](#history)). + + + +## Now — the CLI MVP comes first + +On **2026-10-26** the maintainer paused Wave 3 of Phase 2.5.5 after Step 9b.2 (accepted at `563402e6`) and moved +the work that a CLI user feels to the front: features and bugfixes in `apps/cli` and `packages/core` that +directly affect an MVP. **Paused means pending, not declined:** every paused item keeps its phase-file home, +its recorded acceptance rules and its security sittings, and the [after-MVP queue](#pending-after-the-mvp-not-declined) +keeps its internal order. On **2026-11-04** the maintainer ruled D58–D72 ([decision table](#wave-gating-decisions)) +from a multi-agent review of the roadmap documents and the code at `5a328bca`; the +[priority reset record](phases/phase-2.5.5-hardening-and-remediation.md#mvp-priority-reset--2026-11-04) holds the +evidence summary, the recorded Wave 3 splits and the process amendments. + +### Status at a glance + +| Area | State | +|---|---| +| Phase 1 — Engine and LLM | Complete (M2, 2026-06-21) | +| Phase 2 — CLI | Feature-complete; `relavium@0.1.1` published on npm 2026-06-28 (M3) | +| Phase 2.5 — CLI Consolidation | Complete (M2.5-4, PR #69, 2026-07-08) | +| Phase 2.6.5 — Core reliability | Complete with approved deferrals, 49/51 (PR #92, 2026-10-11; [merge checkpoint](phases/phase-2.6.5-core-reliability-remediation.md#w8-merge-checkpoint--2026-10-11)). CR-70, CR-93 and CR-95's long half stay deferred in [deferred-tasks.md](deferred-tasks.md) | +| Phase 2.5.5 — Hardening | Waves 0–2 merged (Wave 2: PR #94, 2026-10-15). Wave 3 Steps 1–8 merged (PR #95, 2026-10-18); PR95, 9a.1, 9a.2a, 9a.2b, NI, 9b.1 and 9b.2 accepted on `development`. **Steps 9c–19 paused** (25 of 32 increments). NI's Windows clause closed on 2026-10-26 | +| Phase 2.6 — Conversational authoring | 2.6.C done; 2.6.F done except its release clause; 2.6.Q P1–P5 landed; the other fourteen workstreams have not started in their core. The MVP takes its first slices (below) | +| Phases 3–7 | Not started — desktop, VS Code, then Product Phase 2 (managed inference, cloud) | +| Publication | npm serves `0.1.1`. Nothing newer can be published yet: Step 9a.1's interim guard on `development` refuses every artifact until D59's superseding release ADR is Accepted | +| Branches | `development` is ahead of `main` (`6d2a7372`). `main` lacks the PR95 scrubber fixes, NI and the interim guard; a scoped `development`→`main` checkpoint follows this reset (D61) | + +### What a CLI user meets today + +The engine is not the bottleneck: workflows author, run, gate, resume and retry well. The agent and presentation +layers are. The built-in chat agent can only read (`read_file`, `list_directory`, `git_status`) and starts in +`ask`; there is no edit, search or find tool; `run_command` is refused before any approval unless pre-allowlisted; +`relavium run` never prints the workflow's outputs or the run id; replies render as raw markdown; a resumed chat +opens on an empty viewport; a non-Anthropic first key leads to an opaque credential error. The +[priority reset record](phases/phase-2.5.5-hardening-and-remediation.md#mvp-priority-reset--2026-11-04) cites the +code for each. + +## Next work — the CLI MVP + +**The MVP (D58) is `relavium` v0.2.0 on npm** — new global milestone **M3.5** ([README.md](README.md#global-milestone-spine)): +`apps/cli` plus the core engine (`packages/llm`, `shared`, `db` and `mcp` where they decide user-visible +behaviour). A new user installs it, onboards with any of the four providers, chats with an agent that reads, +searches, edits and runs commands under approval with diffs, authors a workflow conversationally or by export, +runs it and sees its outputs, browses and inspects past runs and sessions from the Home, manages providers and MCP +servers there, reaches the web under the chat egress authority, cancels and resumes chats and gated runs, and reads +docs that describe what ships. English only (the `en`+`tr` bar binds Phase 2.6, not the MVP). There is no npm +pre-release before v0.2.0 (D70). + +**Scope lines kept** (this page may not move them): media input, file-snapshot undo and the workflow-run +`egress`/`os` arms stay out ([phase 2.6 scope](phases/phase-2.6-conversational-authoring.md#explicitly-out-of-scope--phase-3--later)); +the MVP only adds a pre-flight refusal for the latter. Local models stay out by product constraint. + +**How increments are accepted (D64, tiered):** an increment that touches an approval or consent path, egress, +keys or secrets, filesystem writes, a new terminal sink for untrusted content, the lease or resume paths, or +release and publication keeps the 2026-10-23 bar (rigor pass, one NEW complete cumulative round with no blocker, +high or medium finding, and a separate security sitting). Other UX, rendering and command increments are reviewed +per milestone: author self-review, `pnpm run ci` plus `pnpm coverage:enforced`, and one independent scoped round +with no high or medium finding; lows go to [deferred-tasks.md](deferred-tasks.md). + +Each milestone ends green on `development` with its canonical docs updated. Items cite their one home: a phase +2.6 bullet as ` · `, a new item as `MVP-A` in phase 2.6's +[CLI MVP additions](phases/phase-2.6-conversational-authoring.md#cli-mvp-additions--2026-11-04), or a recorded +Wave 3 split as `row ` in the [increment table](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments). + +0. **Gate 0 — decide and re-order (no product code).** This reset; the `development`→`main` checkpoint (D61); + drafts for the new ADRs: the pre-1.0 release path (D59), the toolbelt plus tool-render/approval-preview contract + (phase 2.6 ADR row 9, split so `ask_user` comes later), the in-house markdown architecture (ADR row 11, part 1; + no dependency), the ADR-0057 amendments (default grant and `default_mode`, D62; denial feedback, D67), the + keymap (D68), the `run_command` approval card (D63) and the ADR-0095 amendment for CR-71 (D67). Maintainer + gates: CI provider keys for the live-nightly lane (D37), provider spend for a manual three-provider turn and a + Windows machine for MVP-10. A stale-entry sweep of [deferred-tasks.md](deferred-tasks.md) and the phase 2.6 + bullets that 2.6.5 and Wave 2–3 work already satisfied. +1. **MVP-1 — Run it and see it.** Human run output and the run id (MVP-A1, MVP-A2); 2.6.G · Non-interactive + parity (`list --runs`, `status `); 2.6.G · **Liveness** (`engine.reconcile()` on Home open and at + `run`/`gate` start; `status` stays read-only); 2.6.K · **Validate the gate decision shape against `gateType` + before resume**; 2.6.A · **Run-path tool pre-flight for the unwired egress/os arms** (and no `web_search` in + the `create` placeholder); `/export` into `.relavium/workflows/` (MVP-A3); the default model follows the stored + key (MVP-A4); the interactive output budget (MVP-A7). +2. **MVP-2 — First run, resume and a readable chat.** Onboarding offered by `relavium chat` (MVP-A5); standard + provider env-var fallback (MVP-A6); 2.6.G's resume-in-place **with the prior transcript repainted** (the + [deferred `chat-resume` entry](deferred-tasks.md)); picker-on-omitted-id and `chat --continue`; excess + positional arguments refused (row 10b split); 2.6.E · **Markdown + code-block rendering** (in-house, part 1 — + highlighting stays after the MVP); full-screen word-wrap (#61, moved from Wave 5a); tab expansion (#65, row 15a + split); a failure mark for failed tool calls (MVP-A8); the persistent keyless nudge (#51, row 16b split); the + `agent run` stdin hint (#24, row 17a split); a minimal `relavium init` (MVP-A18). +3. **MVP-3 — The agent can edit.** ADR row 9 and its mandatory security review; 2.6.M · **`edit_file`**, + **`search_files`**, **`find_files`** (with `list_directory`'s silent 200-entry cap made visible); `read_file` + ranges and the spill store (MVP-A9); 2.6.M · `write_file`'s approval diff (#305) and **Tool-call rendering + v2** (sanitized target kept after the turn; assistant text between tool calls kept); 2.6.M · **Default chat + agent grant review** as amended by D62 (MVP-A14) with `[chat].default_mode` (MVP-A13); the 80×24 guard on + both chat surfaces and the Home strip row budget (#63, row 15c split). +4. **MVP-4 — The agent runs commands and keeps its footing.** 2.6.G · **Permission management** as ruled by D63 + (MVP-A15; `!` uses the same card); the default agent gains `run_command`; the keymap, including Ctrl-C at a + pending approval rejecting it (MVP-A12; row 11b split); denials returned to the model (MVP-A10); a failed, + denied or aborted turn kept in context as a content-free interruption marker (CR-71, un-deferred, D67); + configurable, finite turn ceilings with conservative defaults (MVP-A11). +5. **MVP-5 — Context and parity (D71).** Anthropic prompt caching (2.5.5.B · #172); a project instructions file + and environment block (MVP-A16); 2.6.E · **Type-ahead message queue**; `chat --model/--provider` and a one-shot + prompt mode (MVP-A17). +6. **MVP-6 — Conversational authoring.** Accept ADR-0058 and ADR-0060 with their reviews; 2.6.A · + `validateAuthoredWorkflow` back-port and **`nodeCatalogIssue` never resolves the agent's own model**; `$ref` + agent resolution; 2.6.B's wizard and conversational authoring loop, its import gate and the UVP hint + (**Discoverability of the UVP**); 2.6.D `{{ctx.*}}` and `agent run --input`; starter workflows (MVP-A19). +7. **MVP-7 — Web.** G30 (rows 13a–13c, pulled forward; 13c is re-scoped when it starts because Step 12b is + paused); 2.6.M · **`web_search` activation**; web fetch through `http_request` under the chat authority. +8. **MVP-8 — The Home as hub.** Phase 2.6 ADR row 6; 2.6.H's history data layer as far as G needs it; 2.6.K's + shared resume core; 2.6.G's `/agents`, the actionable strip, `/workflows` (defined, history, run detail) and the + bound-agent view; Home `/help`; `logs --follow/--failed`. +9. **MVP-9 — MCP and provider management.** Phase 2.6 ADR row 7; MCP servers attached to the default agent under + ADR-0084 consent (MVP-A21); 2.6.I · **`/providers`**, `/mcp`, **`relavium mcp list/add/remove/test`** and MCP + header auth; the masked `provider set-key` prompt (MVP-A20). +10. **MVP-10 — Ship v0.2.0.** The D59 release path (superseding ADR, guard change, security sitting); the final + `development`→`main` checkpoint; release prerequisites and checks (MVP-A22, MVP-A23); v0.2.0 closes 2.6.F's + release clause and M2.6-1. + +**After v0.2.0, first:** v0.2.1 — cross-turn tool memory (CR-70, on its recorded trigger) designed together with +ADR-0096 §5's in-turn overflow continuation (D67). + +**Recorded risks (maintainer rulings):** the published package grants no usage right under `LICENSE` §2 and +stays so (D60); the published `0.1.1` still prints unsanitized model output in `relavium run` (#56, fixed on +`development`) and is left as is because the product has no users yet (D72). + + + +## Pending after the MVP (not declined) + +> **Temporary section — delete it when Phase 2.5.5 and Phase 2.6 close.** It owns **ordering only**; every +> item's definition, size, file list and finding ids live in its phase file, and items are cited as +> ` · ()`. The waves below keep the order derived on +> 2026-07-26; since 2026-11-04 they run **after** the MVP above. The original plan's baseline, axes, Waves 0–1 +> and the PR #81 closing list (the only definition of `#W15-1`…`#W15-24`, which code comments cite) are preserved +> verbatim in the [Phase 2.5.5 register](phases/phase-2.5.5-hardening-and-remediation.md#execution-record-moved-from-currentmd--2026-11-04). ```mermaid flowchart TD - W0["Wave 0 — One true baseline
baseline ✅ · CI truth · numbers"] - W1["Wave 1 — Stop the bleeding ✅
3 CRITICALs · cost cap · ADR-0074"] - LEDGER["#W15-1 — realized-cost ledger ✅
ADR-0076 + ADR-0077"] - P265["Phase 2.6.5 — Core reliability
51 CR items · 8 P0 blockers behind ADR-0078–0084
W0–W8 complete · W3 blocker closed in W8
W4 merged — the hostile-MCP class, ADR-0088
W5 merged — media correctness, ADR-0089+0090
W6 merged — authoring correctness, ADR-0091–0094
W7 merged — PR #90 · W8 merged — PR #92
49/51 closed · approved deferrals retained"] - W2["Wave 2 — Shut the doors ✅
PR #94 merged · fs jail · secrets · config trust
2.5.5 EXIT 1–3 satisfied"] - W3["Wave 3 — Clear the ground
god-file decomposition · CLI net"] - W4a["Wave 4a — The spine
2.6.A/D/H/K + 2 ADRs"] - W4b["Wave 4b — Money floor & close-out
2.5.5.B/C/F/H drain · v0.2.0"] - W5a["Wave 5a — Paint before you build
render-v2 · theme · config"] - W5b["Wave 5b — Home becomes the product
2.6.G/I/J · then i18n sweep"] - W6["Wave 6 — Hands, voice, lineage
2.6.M/B/E + 2.6.N foundation"] - W7["Wave 7 — Orchestration & the gate
2.6.O/P · go/no-go → Phase 3"] - W0 --> W1 --> LEDGER --> P265 --> W2 --> W3 --> W4a --> W5a --> W5b --> W6 --> W7 + G0["Gate 0 — decisions, re-order, ADR drafts"] + MVP["MVP-1…MVP-10
run+see · first run · edit · commands · parity
authoring · web · Home hub · MCP · ship v0.2.0"] + R021["v0.2.1 — CR-70 + ADR-0096 §5"] + W3["Wave 3 remainder (paused)
9c–19b minus the recorded splits"] + W4a["Wave 4a — The spine (what the MVP left)"] + W4b["Wave 4b — Money floor & close-out"] + W5a["Wave 5a — Paint before you build"] + W5b["Wave 5b — Home becomes the product"] + W6["Wave 6 — Hands, voice, lineage"] + W7["Wave 7 — Orchestration & the gate"] + G0 --> MVP --> R021 --> W3 --> W4a --> W5a --> W5b --> W6 --> W7 W3 --> W4b - W4b -.->|v0.2.0 release| W7 W4a -.->|file-disjoint lanes| W4b ``` -### Wave 0 — One true baseline - -Collapse the divergence, make the required gate execute the artifact it ships, and fix the review -checklists before ~30 security-gated PRs are reviewed against them. - -1. ✅ **Merge PR #76** (2.6.Q P1–P5, ADR-0071/0072, the three picker bugs, the param self-heal) — **done - 2026-07-26**, along with #77 (this plan + the Phase-2.5.5 opening + `release.yml`'s `G26` ancestry gate). -2. ✅ **Reserve numbers** — **done**, recorded here: migrations end at `0012` and ADRs at `0072` (verified), - so `0013` = **ADR-0074 §4's conservative money columns** (`agent_sessions.total_conservative_microcents` + `session_costs.conservative_microcents`) — the number was freed when D8 ruled against the approval-preview scrub and is RE-USED here rather than left as a hole · `0014` = 2.5.5.C's enum CHECKs (#101) · `0015` = 2.6.H · - `0016` = 2.6.G's pins · `0017` = 2.6.N lineage; **ADR-0075+** for the nine unwritten 2.6 ADRs (`0073` = the history.db migration lock, `0074` = durable conservative budget commitments — both landed in Wave 1). -3. ✅ **One `ci.yml`/`turbo.json`/`tsup.config.ts` PR** — **done 2026-07-29 (PR #80)**, all eight items: - 2.5.5.H · the required gate never runs the compiled binary + undeclared `drizzle` output (#294, #315) → - local `pnpm run ci` vs `ci.yml` divergence (#312 — and `pnpm ci` is shadowed by a pnpm builtin, so the - script was unreachable by the name everyone types) → the coverage-floor **ruling and its implementation** - (#296, #152) → the `(advisory)` labels (#320) → `THIRD_PARTY_EXTERNAL` (G27, #248) → bundle-closure - single-chunk assert (#314) → `sync:models:check` (#317). *(`release.yml`'s ancestry check, `G26`, landed in - #77; its tag-protection half is configured.)* The PR also closed the four blockers from the 2026-07-29 - roadmap review, and its own CI caught two real defects in it — a TS4111 that `turbo run typecheck` does not - cover, and `pnpm ci` being unreachable because pnpm reserves `ci` as a builtin. -4. ✅ **2.5.5.F · three skills' hardcoded foreign-project paths** (#162) — **done 2026-07-29.** Filed as docs, - actually a functional bug: they pointed at `/Users/dev/…/Agent-Organizer/`, so `add-package`'s `mkdir -p` - would have created a tree outside the repo. All three now anchor on `$(git rev-parse --show-toplevel)`. -5. ✅ **One markdown PR** — **done 2026-07-29**: `packages/mcp` added to all five inventory sites (#128, #129, - #153, #163, #254), the reviewer agent's secrets item extended to the shipped CLI floor (#164), the - security-review skill's SSRF step now requires reusing the shared guard rather than re-deriving one (#167), - and the i18n clause settled by the D5 ruling (#74, #260, #134). -6. ✅ **The locale-bar amendment** (D5, `en`+`tr`) and the **snapshot regen** (D3) — **done**, propagated to - all seven locale sites and merged as PR #79 (two price rises taken, nine retirements accepted). - -> ✅ **Wave 0 is COMPLETE (2026-07-29).** The required gate now executes the binary it ships, -> `apps/cli/drizzle/**` is a declared turbo output, and `lint · typecheck · test` + -> `engine coverage floor (llm, mcp)` are both required checks under branch protection. In phase terms this -> closes **8 of 2.5.5.H's 14** tasks and **3 of 2.5.5.F's 20**. **Wave 1 followed and is also complete -> (2026-07-30, PR #81) — see below; Wave 2 is next.** - -### Wave 1 — Stop the bleeding - -All three CRITICALs, plus the states where `max_cost_microcents` is silently not a cap. Four file-disjoint -lanes, any order between them, plus the money hole they left open. - -**✅ Wave 1 is COMPLETE — lanes (a)–(d) on 2026-07-30 (PR #81), lane (e) on 2026-08-09 (`#W15-3`).** - -> **Superseded 2026-08-09 — kept because the correction is the record.** Lane (e) closed with `#W15-3`: -> `/cost --release` makes §1's escape hatch reachable, and `/cost` renders the durability state. The -> retraction below was accurate when written and is no longer; it stays so the history of the false -> completion claim is not quietly erased. -> -> **Retraction (2026-07-30).** This section previously said Wave 1 was complete and ADR-0074 fully implemented. -> That was wrong on lane (e) and I am correcting it rather than leaving it: **§1's release escape hatch is -> exposed on `GovernorWiring` but reachable from no command, REPL action or Home affordance**, and `/cost` never -> renders `conservativeDurabilityBroken`. §1 names this exact state as forbidden — "a single usage-less response -> would permanently shrink a long-lived chat's cap with no way out" — and this PR REMOVED the accidental escape -> a restart used to provide. A section whose compensating bound is unreachable is not implemented, and marking -> it done made four other documents inherit the error. Two independent reviews flagged it; both were right. -> -> Lane (e)'s remaining work, and two defects it exposed that ADR-0074 does not cover, are tracked in -> **[PR #81's closing list](#pr-81s-closing-list-and-the-two-items-that-need-an-adr)** below. Every lane below is verified against the shipped code, not -against its own task text — the check for each is stated inline. Two gaps are recorded rather than closed; both -are named in their lane and marked in the source, and neither is a shipped defect: - -1. the engine-level composition test for ADR-0074 §3's abort-breakable media-job hold (`engine.ts`, at the abort - listener) — the governor's own state machine is mutation-verified, the composition with `#countRunning`'s - termination gate is not; -2. the CLI-layer test that `restoreConservativeCost` is wired through `buildSessionRuntime` → `GovernorWiring` - (`session-host.test.ts`) — `AgentSession.resume` calling the hook IS mutation-verified in `packages/core`; - only the spread that supplies it is unproven. - -- ✅ **(a) `registry.ts`** — 2.5.5.A · **redact tool-approval previews like `sanitizeInput()` already does** - (#91). *(The originally-planned migration 0013 scrubbing already-persisted previews was ruled OUT — see D8.)* *Closed: `previewFor()` now scrubs the DISPLAY copy while `rawPreviewFor()` keeps an - `unredactedPreview` for in-process classification — splitting the two uses, because a whole-string scrub - flipped the protected-path decision and a per-segment one reopened the leak. Verified: `registry.ts` and - `tools/types.ts` both carry `unredactedPreview`.* -- ✅ **(b) `packages/db` chain, strict** — 2.5.5.C · `SQLITE_BUSY_SNAPSHOT` (#100) → async backoff replacing - the blocking `Atomics.wait` (#226) → **the chat-persister safety net** (#228, CRITICAL) → - serialize `runMigrations` (#99) co-landed with the unconditional 0600 self-heal (#28, #33) → - `createClient` typed error + PRAGMA docstring (#104, #105). `deprecationDate` crash guard (G1) is a - standalone one-liner, droppable anywhere. -- ✅ **(c) render/signals** — 2.5.5.I · **terminal-control sanitization to the four boundaries it never - reached** (G34, G44, #56, #57, CRITICAL — one PR at the shared helper, its own security review) → - `SIGTSTP`/`SIGCONT` + onboarding signal-registration ordering (G0, #50). *Closed at all four boundaries the register names, verified by call count: - `RunApp.tsx` (#56) 9, `final-summary.ts` (#57) 6, `render-error.ts` (G34) 4, `clack-prompter.ts` (G44) 4. G0's - run/gate `SIGTSTP`/`SIGCONT` is `render/run-job-control.ts`; #50 is closed by ordering — - `drive-home.tsx` subscribes signals at `:808`, before `runOnboardingWizard` at `:855`.* -- ✅ **(d) money, split across two PRs** — `budget-governor.ts` only: 2.5.5.A · reservation ledger under - `max_parallel` (G38) → re-armed `budget:warning` + projected `thresholdPct` (G39, G47, G49) → **2.6.Q's - strict-cap half**. Then `fallback-chain.ts`/`cost-tracker.ts` only: 2.5.5.B · `maxRetries: 0` + jitter + - `Retry-After` (#276, #279) → narrowed `#emitSuccess` catch + `CostTracker` bounds (#194, #198) + - **2.6.Q's realized-cost half**. - -- 🔶 **(e) ADR-0074 — durable conservative budget commitments (§2–§5 shipped; §1 INCOMPLETE)** · the money hole the other four lanes leave open: - a bounded estimate retained when a provider may already have billed but returned no trustworthy usage was - kept only in memory, so a crash or a resume reopened a strict cap against money that may already be owed. - §5 (a forward-compatible stored-event READ, the precondition for emitting anything new) → §2 (the - `budget:estimate_committed` contract, the governor's durability barrier, the checkpoint fold and the resume - restore) → §3 (freeze an async media job's accepted units + price) → §4 (the session's conservative - aggregate + per-model attribution + migration + transactional persister write). - -> 2.6.Q's cost-cap item is **split**: its task text spans both `budget-governor.ts` and `fallback-chain.ts`'s -> `#emitSuccess` — the same catch 2.5.5.B narrows. The two halves join two different PRs, not one. -> **Closes M2.5.5-1.** -> -> **Outstanding inside lane (e) — restated at close, because §4 changed the facts.** ADR-0074 §1 ties a -> commitment's release to the surface that renders it. -> -> **Rendering: done on all three surfaces.** The run TUI and `relavium logs` shipped with §2; `/cost` shipped -> with §4's fold and reads *estimated, possibly billed*, with `(no completed call)` for a model that has one -> without a completed call. All three are mutation-verified. -> -> **Release: reachable, as of `#W15-3` (2026-08-09).** `/cost --release` on the chat surface — the one that -> renders the amount, which is what §1 ties it to — clears the durable per-model and aggregate columns first -> and the live governor second, so a released commitment does not return on the next `chat-resume`. `/cost` -> also renders the durability state. This note previously said no affordance called it; that was true and no -> longer is. -> -> The reserved `budget:estimate_released` event stays **reserved and unemitted**, and that is a decision, not -> an omission: the session path restores its conservative total from COLUMNS rather than by replaying events, -> so zeroing them is what makes the release durable there. The event is needed only by a workflow-run release -> surface, which does not exist — a run is not long-lived, and `on_exceed: pause_for_approval`'s per-node -> bypass or raising the YAML cap are §1's own analogy. §1's dated note in -> [ADR-0074](../decisions/0074-durable-conservative-budget-commitments.md) carries the same statement. - -### PR #81's closing list, and the two items that need an ADR - -Two independent adversarial reviews of PR #81 (2026-07-30) found that the cap's core safety claim does not yet -hold across a process boundary. - -**Almost all of this belongs to PR #81 itself, not to a later wave** — a first framing filed it as one, which -was wrong. The test is simple: this PR either ADDED the code a finding is in (`background-failure.ts`), or -claimed to have closed the area it sits in (the terminal-safety boundaries, ADR-0074 §1), or REMOVED a -protection without replacing it (§1's escape hatch, which a restart used to provide). Shipping a regression and -deferring its fix is not a schedule decision. - -**Only §A is genuinely separable.** Both of its items need an append-only ADR before any code — one changes what -"spent" means across a crash, the other supersedes a decision ADR-0074 §5 already recorded. Those earn their own -PRs. Everything from §B down is #81's to close. - -Ordered by whether the repo currently states something untrue, then by blast radius. - -**Every item names the check that would close it, not just the defect.** - -> **Status — 24 of 24 closed and MERGED, across two PRs.** Everything marked ✅ below is fixed and -> break-verified with the mutation confirmed applied; the two PRs are recorded separately because the closure -> dates differ: -> -> - **23 items merged to `main` via PR #81** (2026-08-09). -> - **`#W15-1`, the last one, closed 2026-08-10** behind ADR-0076 + ADR-0077 (see §A) and **merged to `main` -> via PR #82** (2026-08-11), together with the first Phase 2.6.5 batch. -> -> §E's six coverage gaps are all closed — `#W15-16`'s composition test fails by TIMING OUT when the abort -> listener is removed, which is the unkillable run reproduced exactly rather than an assertion standing in -> for it. -> -> **`#W15-2` closed 2026-08-09** as [ADR-0075](../decisions/0075-fail-closed-resume-on-an-unreadable-event-log.md), -> amending ADR-0074 §5: a read that feeds a REPLAY refuses when any row was skipped, a read that feeds a -> DISPLAY stays tolerant. It landed first on purpose — `#W15-1` adds a new durable event type, and an older -> binary reading it falls straight into §5's skip path, so this decision governs how that event degrades. -> -> **One gap this closing deliberately left open**, named here rather than implied: ADR-0075's refusal is -> pinned at `gate.ts`'s own catch (where the blocker was) and at `toUserFacing`, but NOT through -> `dispatch.ts` / `specs.ts` to the process exit. A re-wrapping `try/catch` added in either layer reproduces -> the same blocker and leaves both tests green. Closing it needs a `run(argv, io)` drive with a redirected -> HOME and a seeded on-disk `history.db`; no such harness exists for `gate` yet, and the marker sits at the -> test it concerns. -> -> **`#W15-1` is what remains, and its DECISION is now made**: -> [ADR-0076](../decisions/0076-durable-per-attempt-realized-cost-ledger.md) (Accepted 2026-08-09) — an -> additive durable run event recording one settled provider attempt, emitted through ADR-0074 §2's barrier and -> folded into a `run_costs` row at persist time. It landed after ADR-0075 on purpose: a new durable event type -> is exactly the input that decision governs. -> -> **The implementation is staged, and starts here.** In order, because each step is a precondition for the -> next: -> -> 1. `packages/shared/src/run-event.ts` — `cost:attempt_settled`'s schema, its union arm, and -> `parseStoredRunEvent`. Until this lands nothing else compiles. -> 2. `docs/reference/contracts/sse-event-schema.md` — the canonical shape + which of the two cost events is -> authoritative (ADR-0076's last Negative names that drift risk). -> 3. `packages/db/src/run-history-store.ts` — the `applyDerived` arm writing the `run_costs` row in the SAME -> transaction as its event. The `node:completed` delta then telescopes to zero on its own; a test must pin -> that `SUM(run_costs) == runs.total_cost_microcents` still holds, since that is the invariant carrying the -> no-double-count claim. -> 4. The attempt boundary and its barriers — **corrected by -> [ADR-0077](../decisions/0077-realized-cost-ledger-uses-the-conservative-commitment-barrier.md)**. ADR-0076 -> §1 asked for an `await` at the settle point; there is no `await` to be had there. The seam's observer is -> `onAttempt?: (record) => void` — synchronous, void — and BOTH money events are emitted from that one -> callback a few lines apart, which `budget-governor.ts` already states outright ("which cannot await"). So -> the ledger takes ADR-0074 §2's shape: **start** the write at the settle instant on a chained in-flight -> promise, **join** it at three barriers — before the next egress admission, **before tool dispatch** -> (`agent-turn.ts`'s `await dispatchToolCalls(...)`, the barrier this adds beyond §2), and at the turn/node -> terminal. Every barrier awaits AND observes the failure, because `#emitDurable` is total for store faults -> and resolves. -> 5. `packages/core/src/engine/checkpoint.ts` — **corrected during implementation**. The instruction here was -> "fold it as a SUM of deltas", borrowing ADR-0074 §2's reasoning, and the wrong half of that cost two -> rewrites. Summing into the realized total double-counts (a node terminal's snapshot already contains its -> attempts); summing into a separate accumulator and maxing the two families UNDER-counts (a media node -> writes a snapshot and emits no attempt row, so every attempt after the last boundary vanishes). What is -> correct is the fold already two arms above it: `Math.max` over every durable ABSOLUTE total. §2's -> rejection of `Math.max` does not carry — it was rejected there only because a release can DECREASE a -> conservative total, and realized spend is monotonic. -> -> **STATUS: all five steps are landed** (`8d7ffcf` … `50c60bd`), each with an Opus and a Sonnet round folded. -> The break-verify that matters most was step 4's ordering, and it now holds: with the tests draining to -> quiescence before asserting, deleting B2 reddens the tool-dispatch test and deleting B3 reddens the node- -> terminal test. B1 is masked by B2 on the current fixture (the second egress only comes after the tool), so -> it is covered by construction rather than by a red — named rather than implied. -> -> **This is now the prerequisite of Phase 2.6.5, not a parallel track.** Its five steps touch the same four -> files `CR-10` (ordered append tail) and `CR-12` (effect journal) restructure, so it lands first — see the -> 2.6.5 section below. -> -> **What ADR-0076 explicitly does NOT close**, so it is not read as more than it is: the cost of a TOOL EFFECT. -> A resumed run can still re-execute an `http_request` POST, a `run_command` or an MCP mutation — the -> duplicate EFFECT is the harm, and no cost bookkeeping prevents it. That needs a durable effect journal with -> a prepare/receipt pair, which is a different decision about a different failure and earns its own ADR. It is -> the natural next one after this chain (forward-compatibility → conservative commitment → realized cost). -> -> Two decisions inside the closed set are worth surfacing here rather than leaving in a commit message. -> `#W15-4` **changed a previously-tested behaviour**: #228 pinned that a session survives a failed durable -> write and the next turn is clean; it now stops instead, because continuing to spend above a transcript that -> has fallen behind is the defect. And `#W15-23` is scoped to `run-history-store` only — the other -> `packages/db` stores share the same outer-`db`-inside-a-transaction convention, which remains open, is -> stated here rather than implied closed, and has its own PR queued after this one. - -#### A. Needs an ADR first — the only part that is NOT PR #81's to close - -- ✅ **`#W15-1` (blocker) — a workflow's REALIZED cost is not durable at the provider-attempt boundary.** - *Closed 2026-08-10 by ADR-0076 + ADR-0077 and the five staged steps; the barrier is a chain-and-join, not - the inline await ADR-0076 §1 first specified. See the staged plan below for what each step landed and the - three carried-forward gaps.* - `engine.ts` folds `cost:updated` into memory and streams it; the store documents that it is never persisted, - and the checkpoint can only recover cost from a LATER `node:completed`/gate snapshot. So: a paid call - succeeds → the model asks for a tool → the process dies during the tool → the realized cost is gone and the - resumed node re-runs the same paid call, against a cap that has forgotten the first one. ADR-0074 makes only - the CONSERVATIVE commitment durable; a call that returned trustworthy usage has no equivalent barrier. - *Needs an ADR: an idempotent per-attempt durable ledger, written before the next tool side-effect, the next - egress and the turn/node terminal. Not a local fix — it changes what "spent" means across a crash.* -- ✅ **`#W15-2` (blocker) — a tolerant read is fail-OPEN on the resume path.** ADR-0074 §5 drops an event whose - `type` an older binary does not know, and `checkpointer.ts` builds resumable state from what remains. An - older binary cannot know whether the dropped row was a node terminal, a job submission, a gate decision or a - cost commitment — so it may re-run completed or already-submitted work. Preserving the sequence high-water - mark prevents a UNIQUE collision; it does not restore lost state. Tolerance is right for `logs`/`status` - (read-only); it is wrong for a replay that changes the world. - *Needs a superseding ADR (§5 is append-only): fail closed on the resume path with "a newer Relavium is - required" when any row was skipped, while read-only surfaces stay tolerant — or classify events as - state-bearing vs not, and tolerate only the latter.* - -#### B. PR #81 — ADR-0074 §1, the half that makes the rest safe - -- ✅ **`#W15-3`** (blocker) — the release is exposed but unreachable, and nothing renders `durabilityBroken`. - `GovernorWiring.releaseConservativeCommitments` and `conservativeState()` exist and are tested; no command, - REPL action or Home affordance calls either. §1 requires the surface that RENDERS the amount to also expose - clearing it, and this PR removed the accidental escape a restart used to provide. - *Closed by `/cost --release`: `/cost` renders the durability state, and the release clears the per-model AND - aggregate columns before the live governor, so it survives `chat-resume`. The durable - `budget:estimate_released` named here was NOT needed — the session path restores from columns, not from an - event replay — so it stays reserved; see the lane-(e) note above.* - -#### C. PR #81 — data integrity, each with a concrete failure - -- ✅ **`#W15-4`** (blocker) — session persistence failures are invisible to the producer. `RunEventBus` isolates - listener errors by design, so a persister that cannot write still lets the turn complete. Two consequences: - `persister.ts` advances the in-memory cost total BEFORE the DB write, so a failed `recordSessionCost` leaves - the durable total low and a resume restores a cap that has forgotten real spend; and a failed `commitTurn` - leaves the staged turn uncleared, so the next `beginUserTurn` overwrites the previous user message and the - durable transcript loses an exchange the in-memory session still has. - *Closes when: cost and transcript persistence move to an awaited durability port (UI subscribers may stay - passive), and a persistent write failure puts the session in a durability-degraded state that refuses new - egress rather than reporting success.* -- ✅ **`#W15-5`** (high) — stored payloads are not checked against the authoritative columns. `readEventLog` - selects `seq`/`event_type` alongside the payload but never compares them to the parsed event's own `runId`, - `sequenceNumber` and `type`. A row with `seq = 2` carrying `sequenceNumber: 1` is accepted, leaving the - high-water mark wrong and the resume colliding on the next write. - *Closes when: every denormalized projection is compared after parse and any mismatch is a - `CorruptRunEventError`.* -- ✅ **`#W15-6`** (high) — terminal cost snapshots on failed/cancelled runs are dropped by the read models. - `node:failed`, `run:failed` and `run:cancelled` all carry cost in the shared schema, but the store folds - none of them, the checkpoint folds none, and the run TUI's summary ignores the terminal figure. A sibling's - paid media cleanup after a root failure exists ONLY on `run:failed.cumulativeCostMicrocents`, so replay and - history lose it. Pre-dates this PR, but the PR touches all three files and claims cost safety. -- ✅ **`#W15-7`** (medium) — `media_job:submitted` accepts a half-frozen basis. `units` and - `acceptedCostMicrocents` are independently optional, so a row with a frozen cost and no frozen units resumes - with the old reservation and a re-derived volume. The invariant is not "both or neither" (the approved-bypass - case legitimately has units only) but **`acceptedCostMicrocents` present ⇒ `units` required**. - -#### D. PR #81 — security and correctness - -- ✅ **`#W15-8`** (high) — the new failure-reporting paths can leak credentials. `background-failure.ts` prints an - arbitrary rejection message and, under `RELAVIUM_DEBUG`, a raw stack; `sanitize.ts` strips ANSI/control/bidi - but performs no secret redaction. Same class in the session listener notice, `render-error.ts`'s verbose - stack and `index.ts`'s fatal debug stack. The rejection can be an MCP error, a provider response body or a - tool result — all of which carry keys, `Bearer` headers or credentialed URLs. Existing tests plant ANSI, never - a secret. - *Closes when: the shared secret-shape redactor runs BEFORE terminal sanitization on every one of those paths - (stacks included), with tests planting `sk-*`, `Bearer`, `Basic`, credentialed URLs, query tokens and PEM.* -- ✅ **`#W15-9`** (high) — the stream path can throw out of a contract that says it never throws. - `#runStreamAttempt` calls `#emitSuccess` OUTSIDE its try/catch (the `generate()` path has it inside), so a - provider returning non-integer usage makes `assertAccountableUsage` throw raw out of the async generator — - and `streamOneTurn`'s `for await` has no try/catch, so it crashes the turn unclassified, AFTER the content was - produced and paid for. -- ✅ **`#W15-10`** (high) — `--json` output is not sanitized. `status --json` / `gate list --json` carry - model-controlled text; `JSON.stringify` does not escape Unicode bidi overrides (CVE-2021-42574). The human - render path is sanitized; the JSON branch is not. `createJsonRenderer` is the same gap on the NDJSON path. -- ✅ **`#W15-11`** (high) — `createPlainRenderer` sanitizes AFTER splitting on newlines, so an embedded newline - still forges a line. Every other boundary this PR touched sanitizes first; `renderer.ts` is the exception. -- ✅ **`#W15-12`** (high) — `isBlankPreview` treats a fully-redacted preview as informative (`chat-mode.ts`), so a - user pressing "always" on `Approve write to [redacted]?` auto-approves every `write_file`/`run_command`/ - `http_request` for the session, having been shown nothing. -- ✅ **`#W15-13`** (medium) — the config reader chmods PROJECT configs to 0600. `reassertOwnerOnly` runs on every - layer, but `config-spec.md` says project files are committed and shared. Reading a config mutates repo file - permissions, and `statSync`/`chmodSync` follow symlinks. Self-heal belongs to the canonical global config - only. -- ✅ **`#W15-14`** (medium) — a provider `Retry-After` sleep is not abort-aware. `fallback-chain.ts` passes the - delay straight to an injected `sleep` with no signal, and the real hosts use a plain `setTimeout`. A cancel - during a 60 s clamped wait hangs until the timer fires. The existing abort test resolves `sleep` immediately, - so the window is untested. The docblock also says an oversized value becomes `undefined`; the code clamps. -- ✅ **`#W15-15`** (medium) — corruption is reported to a multi-run view as "no data". `readPerRunOrDegrade` - returns only the fallback, so `gate list` can print "No pending human gates." for a run whose gates were lost - to corruption. It must return the degradation alongside the value, with a warning in human output and a - machine-readable marker under `--json`. - -#### E. PR #81 — coverage gaps it shipped knowingly - -All six are now CLOSED and their in-source markers removed. None had been a shipped defect — each had -mutation-verified coverage one layer down — but closing them surfaced one that was: `onLegacyMediaJobHold` -reached `WorkflowEngine` from a real `gate.ts` sentence and the constructor never read it, so ADR-0074 §3's -hold notice was dead and a `relavium gate` resume of a legacy parked media job stalled silently. Found by -the review of `#W15-16`, fixed with it, and now the signal that test drives. - -- ✅ **`#W15-16`** — the engine-level composition test for §3's abort-breakable media-job hold; deleting the abort - listener leaves the whole suite green. -- ✅ **`#W15-17`** — the CLI-layer test that `restoreConservativeCost` is wired through `buildSessionRuntime`. -- ✅ **`#W15-18`** — the `coalesce` backfill of `session_costs.model_catalog_id` (needs a `model_catalog` FK row - the block does not seed). -- ✅ **`#W15-19`** — that a chat turn's terminal is emitted only AFTER `flushBudgetCommitments` resolves. -- ✅ **`#W15-20`** — the H3 approved-bypass `acceptedCostMicrocents` omission; reverting it leaves the suite green. -- ✅ **`#W15-21`** — `agent-run.ts`'s `attachConservativeWriter`; deleting it silently breaks every capped - `agent run`. - -#### F. PR #81 — standards and documentation drift - -- ✅ **`#W15-22`** (low) — the PR's "no unsafe `as`" conformance claim is false. Two assertions in new production - code: `shared.ts`'s `readRetryAfter` casts an arbitrary `headers.get`, and `openai.ts`'s `mapOpenAiApiError` - casts a `headers` field back onto a parameter type that lacks it. A structural `headers?:` plus runtime - narrowing removes both. -- ✅ **`#W15-23`** (medium) — `persistEvent`'s `applyDerived` uses the outer `db`, not the transaction handle. - Harmless under `better-sqlite3`, but it breaks ADR-0005's Postgres-portability promise on the highest-volume - money write. -- ✅ **`#W15-24`** (low) — canonical docs describe code that was not shipped. `tool-registry.md` says path - redaction is per-segment (it is whole-string, and the same file says so correctly three paragraphs later); - `llm-provider-seam.md` still claims model-discovery/media-poll keep a small SDK retry; ADR-0028's amendment - note still says ADR-0074 §2–§5 have not landed. Each could send a future reader back toward a #91-class hole. - -### Phase 2.6.5 — Core reliability remediation (between Wave 1 and Wave 2) - -Three independent core reviews of the post-Wave-1 tree converged — independently — on the **same seven blocking -gaps**: effect journal, stdio MCP consent-before-spawn, run lease, compaction trust elevation, stream grammar, -input admission, event-log ordering. Three separate reviews landing on the same seven points is not opinion. - -The full, self-contained work list is -[phase-2.6.5-core-reliability-remediation.md](phases/phase-2.6.5-core-reliability-remediation.md) — **51 items** (`CR-01`…`CR-98`; `CR-96` and `CR-97` were opened 2026-09-14 from the `W7` ADR reviews, `CR-98` on 2026-09-18) with evidence, fix, acceptance criteria and a decision/ADR/gate register, written so the work -can be done from that document alone. An adversarial plan review on 2026-08-10 corrected the phase boundary, -the exit rule and the execution order, and added two items (`CR-17` resume identity, `CR-63` `input_schema` -docs-only); `CR-64` came from the Batch 1 triage, `CR-21b` from ADR-0082 §10 and `CR-21c` from the `W2` -document review on 2026-08-25, which is why the total has moved since the list was first written. - -> **Historical phase checkpoint — 41 of 51 closed (updated 2026-09-18; at W6's merge it was 39 of 48 — see the register for why not 43: W5 was counted twice). `W0`–`W2` merged clean; `W3` merged 2026-08-30 (PR #86) with one reproduced BLOCKER and nine verified findings open — see the `W3` residuals in [deferred-tasks.md](deferred-tasks.md). `W4` MERGED 2026-09-01 (PR #87) behind [ADR-0088](../decisions/0088-the-mcp-boundary-is-hostile.md): a systematic review of the branch found five merge blockers — an orphaned MCP child on a signal, an optional validated `fetch`, a bypassable transport byte bound, discovery paging past its budget, and two cloud-metadata endpoints reachable through the local opt-in — all reproduced and fixed before it merged. `W5` MERGED 2026-09-02 (PR #88) behind -> [ADR-0089](../decisions/0089-media-correctness-four-boundaries.md) + -> [ADR-0090](../decisions/0090-a-continuation-token-rides-the-part-it-belongs-to.md): all six of -> `CR-50`–`CR-55` closed, two of them for one half of a two-part obligation and saying so in their own -> headings. A systematic maintainer review of the branch returned **six merge blockers** — the async -> media-job result bypassing the first-resolution pin, a run-scoped re-host ceiling that reset on resume, a -> capability gate keyed on the model id alone (breaking custom `base_url`), a negative DB media rate buying -> strict-cap headroom, a load-bearing migration backfill with no upgrade test, and a drifted canonical -> AgentRunner spec — all reproduced and fixed before it merged. -> -> **`W6` — authoring correctness — MERGED 2026-09-04 (PR #89)** — `CR-60`–`CR-64` behind -> [ADR-0091](../decisions/0091-first-means-first-declared-not-first-to-finish.md)–[ADR-0094](../decisions/0094-a-tool-grant-is-checked-when-the-plan-is-built.md), -> 32 commits — 27 of work over seven internal review rounds, then five folding four maintainer rounds on -> PR #89: `first` means first DECLARED and the plan stops claiming otherwise; a -> tool grant is refused when the plan is built rather than mid-run; an expression sees only its transitive -> closure and a literal out-of-closure read is refused at parse; an `output_schema` is compiled at parse in an -> allowlist-strict mode and enforced at run time — with **no new dependency**, because the compiler the -> deferral said we needed already existed one package away. -> -> **What the wave is actually worth recording for is that most of what the review rounds found were defects in -> the FIXES, not in the code they repaired.** The branch-order search was wrong three times in three rounds, -> each time in the mirror of the previous error. The expression scan's central soundness claim — that it never -> produces a false refusal, which is the entire justification for a regex over a parser — was asserted in three -> documents and false in EIGHT distinct ways — the eighth, a Unicode identifier escape, found after the -> wave was declared complete. One stand-down gate, added to prevent a false refusal, silently -> removed the whole check it was protecting. Every one is a dated correction inside its own ADR rather than a -> quiet rewrite, because the shape of the error is the reusable part. -> -> **The maintainer review then found two merge blockers, and both broke a headline claim of the wave.** Deep -> validation did not enforce `required` when the property's schema constrained nothing — `{ properties: { -> status: {} }, required: ['status'] }` accepted `{}`, because an unconstrained property compiles to -> `z.unknown()`, which is optional inside a `z.object`, and presence was checked only for names ABSENT from -> `properties`. And the expression scanner's regex was quadratic and synchronous: 514 ms at 32 K of -> whitespace, on a parser that accepts 2 MiB, with plan build synchronous so no deadline or cancel could -> interrupt it. Both reproduced and fixed before the merge, with eight further High findings — among them a -> crash on `{ type: 'string', format: 'email', maxLength: 254 }`, which is the exact shape the new canonical -> subset document tells authors to write, caused by an `as unknown as` cast that CLAUDE.md rule 1 forbids -> precisely because it turns a type error into a runtime one. -> -> **Eleven residuals** are recorded in [deferred-tasks.md](deferred-tasks.md) — ten under the four `W6` -> residual headings plus ADR-0091's zero-branch-merge question, filed with the fan-in items. Two await a -> maintainer ruling. -> -> **The `W4`-waits-for-the-`W3`-blocker gate was lifted deliberately on 2026-09-01, by the maintainer, and it is recorded here rather than left to be inferred from the commit order.** The reasoning: the blocker is a `BoundedEventStream` drop on a stream nobody iterates, its fix is ADR-0087 §1 — a decision that was still **Proposed** and unapproved — and `W4` touches a disjoint surface (the MCP boundary). Blocking a whole wave on an unapproved ADR would have stalled the phase rather than protected it. **The blocker stays open and stays first in the `W3` residuals**; nothing here closes it or reduces its severity. **Updated 2026-09-04:** ADR-0087 is now **Accepted**, with nine dated corrections written into it after a review refused acceptance. That removes the "unapproved ADR" half of this gate — §1 is now a settled decision — but §1 remains **unimplemented** and the blocker is unchanged. Its Correction 1 records what §1 must settle before it can be built: the two factories §1 names are not symmetric — a host calls `createSessionHandle` directly, but a `RunHandle` is only ever produced inside `WorkflowEngine.start()` / `resumeFromCheckpoint()`, so the run half needs the mode plumbed onto an engine entry point before it can be passed at all. A `subscribe-only` handle must also not present a silently-empty iterable. -> -> - **Batch 1, merged 2026-08-11 (PR #82)** — the prerequisite (`#W15-1`), the oracle (`CR-90`, `CR-91`) and -> all of `W0` (`CR-01`–`CR-03`). `CR-64` was added in the same batch and shipped 2026-09-02 with `W6`. -> - **Batch 2, merged 2026-08-24 (PR #83)** — all of `W1`: the eight P0 blockers plus `CR-92`, behind -> ADR-0078…ADR-0084. A comprehensive review of the assembled PR found seven further defects, six of them in -> the W1 code itself; all seven were fixed and mutation-verified before merge. -> -> - **Batch 3, merged 2026-08-28 (PR #85)** — `W2`, liveness and deadlines, behind -> [ADR-0085](../decisions/0085-the-node-executor-owes-liveness-and-the-engine-enforces-it.md). All six -> items closed; the wave's own closing register is in the phase document. The assembled PR drew a -> **Request changes** verdict with fourteen findings, five of them blockers, plus a later round of eight; -> all were verified against the code before being acted on, and the register records which were fixed and -> which were declined with the reason. The five blockers had one root cause — an ownership decision taken -> at a point in time, guarding work that spans an await. -> Its document review moved the count twice, and the two moves **cancel**: `CR-21` turns out to have -> shipped **with `CR-14`** in PR #83 (ADR-0082 decides both, and the execution graph above already -> scheduled them together) and only its heading was stale, while `CR-21c` — an unbounded individual -> `pollMediaJob` call — was found during the same review and added. `W2` was therefore still **five** open -> items out of six, not four; the phase total moved 47 → 48, not the wave. All five then closed on -> 2026-08-27 — `CR-22`, `CR-21b`, `CR-21c`, and finally `CR-20` and `CR-23` together, which -> [ADR-0085](../decisions/0085-the-node-executor-owes-liveness-and-the-engine-enforces-it.md) decides as -> one because they are one mechanism at one place. -> -> **Not to be confused with the 2.5.5 remediation's own `W2`** ("Shut the doors", in the graph above). Same -> label, different phase: this one is 2.6.5's liveness wave. -> -> Per-item history, the seven post-review findings and the carried-forward gaps live in the phase document. - -> **Pre-`W7` close-out, 2026-09-06.** *(Superseded in part on 2026-09-14: `CR-96` and `CR-97` reopened exit criterion 1, -> the count is 41 of 51, and `CR-94` is decided and scheduled into `W7` — see the next block.)* The last two non-deferrable items closed, so **exit criterion 1 is met** -> and the phase count is **41 of 48** (40 ✅ headings plus `CR-95`'s short-term half; the arithmetic is shown -> because it has been wrong twice here). `CR-73`: `invoke_agent` was advertised to every model granted it while -> its delegate is wired nowhere in the tree, so the advertise-filter now reads a tool's declared -> `requiresDelegate` and not only its `ToolHost` arm — `read_media` is dropped for the same reason, which is -> `CR-50`'s own argument for leaving it unwired. `CR-80`: a rejected custom `base_url` was caught and the -> DEFAULT adapter left standing, which is the official API; it now installs a refusing adapter that names the -> URL's shape and never its credentials. **Criteria 5 and 7 also closed**: the three missing security sittings -> (prompt/trust provenance, hostile MCP, provider/config trust) and the `W5` and `W6` closing registers are -> written, each stating that it was written after its wave merged rather than with it. **Criterion 6** passes -> on both halves, read from the exit code — `pnpm run ci | tail` reports `tail`'s status and produced one false -> green in this session. Still open: criterion 2 (`CR-94` has no deferral record, and that is a maintainer's -> call) and criterion 4 (the `W3` live blocker keeps the gap-free stream promise wider than the code). -> *(Criteria 2 and 4 have both moved on: `CR-94` was decided 2026-09-13 and its remainder now has a deferral -> record, and ADR-0087 §1 — criterion 4's blocker — is scheduled into `W8`. Criterion 5, called closed above, -> REOPENED on 2026-09-18 with a fifth sitting, `history.db` at rest.)* - -> **`W7` decisions made and accepted, 2026-09-14.** -> -> **What was decided.** The maintainer settled `CR-70`–`CR-72` and `CR-94` on 2026-09-12 and 2026-09-13. Four ADRs -> were **accepted** on 2026-09-14, with their implementation staged in `W7`: -> -> - [ADR-0095](../decisions/0095-what-an-agent-session-remembers-across-turns.md) decides that tool history is -> persisted as structure, never content; that carrying it into the model's context is deferred; and that `memory` -> is implemented. `memory` stays inert until `W7` lands. -> - [ADR-0096](../decisions/0096-a-request-is-measured-before-it-is-sent.md) decides that a request is measured -> before it is sent, that a context overflow is classified and recovered only before any tool runs, and that input -> is priced. Until `W7` lands, the estimate still prices output only. -> - [ADR-0097](../decisions/0097-a-budget-approval-is-an-allowance-not-an-exemption.md) decides that a budget -> approval is a dispatch-owned allowance, shown before it is granted. Until `W7` lands, an approved step still runs -> uncapped. -> - [ADR-0098](../decisions/0098-a-session-effect-row-holds-no-result-and-never-replays.md) decides that a session's -> effect row holds no result and never replays. -> -> **What the reviews found.** The first six review rounds shaped the ADRs; a seventh, on 2026-09-18, reviewed them -> against the tree before any code (see the block below). The six surfaced two shipping defects, now -> register items `CR-96` and `CR-97`. Both meet the phase's non-deferrable definition, so exit criterion 1 is open -> again, and the count is **41 of 51**. -> -> **What `W7` now covers:** `CR-71`, `CR-72`, `CR-98` (ADR-0096's prerequisites, opened as an item 2026-09-18), -> `CR-94`, `CR-96` and `CR-97`, plus `relavium budget resume`, which moves here from 2.6.K because ADR-0097's -> shown-amount guarantee otherwise has no non-TTY surface. Five canonical claims that were already false are -> narrowed ahead of the code: -> -> - the session spec's tool history and export `tools` union; -> - `memory`'s default; -> - the uncapped approved budget step; -> - the SSE schema's "user/assistant/tool messages are persisted" sentence (2026-09-18); -> - the chat reference's "full transcript" in the export (2026-09-18). -> -> *(This `W7` is Phase 2.6.5's seventh wave. It is not the remediation's "Wave 7 — Orchestration & the gate" in -> the graph at the top of this file, exactly as `W2` is not Wave 2.)* - -> **`W7` document review, 2026-09-18 — before any code.** Eight dimensions reviewed the four ADRs, the register, -> the deferrals and every canonical document they touch against the tree; each finding was verified by an -> independent adversarial pass. 113 findings held, and the maintainer settled seventeen decisions the same day. -> Every ADR carries a dated note; the register carries a per-document landing checklist. Three document defects -> mattered most, and all three would have been discovered mid-implementation: -> -> - **ADR-0098's join does not exist.** It says the effect journal already records the tool-call id; every session -> surface writes a wiring constant instead, so `W7` must thread an engine-assigned id through the dispatch port. -> - **ADR-0098's monotonic turn key could not be derived** from the journal, because the sweep deletes the rows the -> high-water mark would be read from. It becomes a durable column on the session row. -> - **ADR-0096 §1's per-part floor** would have charged an inline image megabytes of tokens, contradicting the -> per-modality media ceiling in the same section. -> -> Two of the seventeen decisions change what ships beyond those ADRs: `history.db` opens with -> `PRAGMA secure_delete = ON`, so "no tool result at rest" is true of the file's bytes and not only of the row; and -> a pre-content HTTP 4xx now releases its admission instead of committing an input-sized estimate, which keeps a -> rate-limit burst from exhausting a session's cap once input is priced. ADR-0087 §1 — the `W3` live blocker that -> belonged to no wave — is scheduled into `W8`. - -**This is the corrected execution order, and it is what the graph above shows:** - -1. **`#W15-1` first.** Its five staged steps touch `run-event.ts`, `engine.ts`, `checkpoint.ts` and - `run-history-store.ts` — the same four files `CR-10` and `CR-12` restructure. Landing it after the durability - spine means writing its barrier against a persistence path that is about to change. -2. **Then Phase 2.6.5, closing before Wave 2 opens.** Which is only true if the work Wave 2 would block on - lives there — so **the hostile-MCP threat class moves into 2.6.5**: `CR-16` (consent before a stdio spawn) - plus `CR-40`–`CR-42` (transport cancellation, DNS/redirect SSRF, ingress bounds). Wave 2's queue item 2 is - **executed as 2.6.5's `W4`**; the 2.5.5 finding ids keep their home in the 2.5.5 phase doc and certify from - there. Wave 2 keeps the MCP name-collision/tool-poisoning trust items, the fs jail, the secrets layer, - persistence-security and the certification. -3. **Within 2.6.5, the oracle comes before the spine.** `CR-90`/`CR-91` (crash + durable-truth harness) land - first because they are the instrument `CR-10 → CR-11 → CR-92 → CR-12` is proven with; `CR-92` moved out of - the harness group into the spine, since an API returning `completed` while history says `failed` is a - runtime defect, not a test concern. - -Two rules that changed with the plan review: - -- **Nineteen items are non-deferrable** — every W0/W1 item plus `CR-50`, `CR-55`, `CR-73`, `CR-80`, `CR-92`, `CR-95`'s short-term fix, and (added 2026-09-14) `CR-96` and `CR-97`. Each has a cheap fail-closed option (refuse, remove, narrow the claim), so "too big - to fix now" argues for the cheap option, never for deferral. The previous exit criterion permitted deferring - all 43 items and declaring the phase complete. -- **The gate is `pnpm run ci` AND `pnpm coverage`.** `coverage` is a separate required CI check and is not - inside the `ci` script; this phase edits `packages/core` heavily, and the CI coverage job does not block on - `core`. - -`CR-01`–`CR-03` were half-closed by Wave 1 and finished right after the oracle — **all three are closed as of -2026-08-11** (PR #82). `CR-03` was a propagation gap from `#W15-10`'s own fix: the finding named three `--json` -paths that never got the safe serializer, and closing it found **five**, which is why the call-site half is now -an ESLint selector rather than a list. +**Pulled forward into the MVP (the waves below keep the rest):** the `engine.reconcile()` wiring (#229), the +gate-decision/`gateType` check (#0) and the run-path tool pre-flight (#37/G29) (Wave 4a → MVP-1); 2.6.A's +back-port and catalog pre-flights, 2.6.B and 2.6.D (Wave 4a/6 → MVP-6); the toolbelt's edit/search/find tools, render v2 slices, the default grant and +`web_search` (Wave 6 → MVP-3/7); 2.6.G, 2.6.H and 2.6.K as far as the Home hub needs them (Waves 4a/5b → MVP-8); +2.6.I (Wave 5b → MVP-9); #61 word-wrap (Wave 5a → MVP-2); #172 prompt caching (Wave 4b → MVP-5); the v0.2.0 +release (Wave 4b → MVP-10). After the MVP, a wave takes up whatever of its items the MVP did not finish. + +**Remaining product lanes after v0.2.1 (phase 2.6, ordered):** sub-agents and the rest of orchestration +(2.6.N → 2.6.O/2.6.P, Waves 6–7); `/rewind`/`/fork` and persisted input history (2.6.E); `todo` and `ask_user` +(2.6.M, ADR row 9 part 2); syntax highlighting (ADR row 11 part 2); settings, themes and `en`+`tr` (2.6.L, +Wave 5a/5b); onboarding v2's account stub (2.6.J); history retention and delete (D33, Wave 4b); scope-line items +only after a phase amendment and an ADR. + + + + + + ### Wave 2 — Shut the doors -**Complete on `main`, 2026-10-15, merged as PR #94 (`ab08209e`).** The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) joins all forty queue IDs, the exit/D sign-off and final correction evidence. Hostile-MCP work is reused from W4/PR #87; the remaining filesystem, secrets/config, persistence and media work is complete within its recorded limits. The queue below preserves scope and traceability. - -1. ✅ **Completed in W4/PR #87:** 2.5.5.D · project-layer MCP name collision redirecting a global secret - (G19) and MCP tool-definition poisoning (#202); see the - [W4 closing register](phases/phase-2.6.5-core-reliability-remediation.md#w4-closing-register). -2. **MCP queue, strict — EXECUTED AS PHASE 2.6.5 `W4`, and COMPLETE there (2026-09-01, [ADR-0088](../decisions/0088-the-mcp-boundary-is-hostile.md)).** The queue below landed in five steps, plus `#202` and `G19` from step 1 of this wave and the MCP half of `#21` from 2.5.5.E — all the same threat class and the same security sitting, so splitting them would have booked the reviewer twice. Certify the 2.5.5 ids from the `W4` closing register in [phases/phase-2.6.5-core-reliability-remediation.md](phases/phase-2.6.5-core-reliability-remediation.md). - *The original queue, for the record:* Connect-phase timeouts on every transport - (#35, G32, #205) → transport/discovery/result size bounds against a hostile server (G33, #201, #209, #288) → - structured `serverId`/`reason` discriminants (#203, #204) → the two fail-loud violations (#206, #207) → - per-file transport-adapter tests (#297) → `CLIENT_INFO.version` (#208). These finding ids keep their home in - [phases/phase-2.5.5-hardening-and-remediation.md](phases/phase-2.5.5-hardening-and-remediation.md) and - certify from there in step 6 below; the *work* happens in 2.6.5 alongside `CR-16` (consent before a stdio - spawn) and `CR-41` (the DNS/redirect SSRF hole), because they are the same code and the same security - sitting. Splitting them across two phases would book the hostile-MCP reviewer twice. -3. ✅ **The fs jail, ONE reviewed change set — complete on `main`, PR #94** — 2.5.5.D · sensitive-credential floor for `.kube`/`.azure`/gcloud - (#36, #39) → extract `deepestExistingReal` to `packages/shared` (#235) → **2.6.M's `project`-tier - `extraRoots`** and **2.6.N's fs-floor home-anchoring**, both *pulled forward*. `~/.relavium/tmp/` (G21) - is resolved under the same general predicate (decision D12) — otherwise `fs.ts` takes two security reviews. -4. ✅ `discoverCatalog` realpath containment (#1, accepted in Step 2); the secrets-layer test pair — OS-keychain bridge (#27) and - `readSecretFromStdin` + stdin cap (#31, #293); secret-typed `run --input` rejection (#25); the unredacted - workflow-snapshot doc + lint (#30); 2.5.5.B · `scrubSecrets` param-name broadening (#196). -5. ✅ Persistence-security: 2.5.5.C · `assertRelativePath`'s Windows drive-relative form (#113), media-store - hardening (#111, #108, #112), media `mimeType` verification (#107, #109, #110), and certification of - W7's structural session/tool-content controls (#115, Wave D15). 2.5.5.A · chat-export continuity/slug - completion and existing tool-content exclusion (#80, #81, #83, D14), plus `analyzeSecretTaint`'s - context sites (G22). 2.5.5.D · the dormant `default_headers` admission/custody gate (G20, D13). -6. ✅ **Phase 2.5.5 exit criteria 1, 2 and 3 and sub-stream D acceptance are satisfied on `main` by PR #94.** The D half of M2.5.5-2 is complete; the I half remains open. - -> Book **four security-review sittings by threat class** (fs/path-jail · secrets-at-rest · secrets-input · -> config-trust), not ~30 per-PR passes. The **hostile-MCP** sitting moved to Phase 2.6.5 with the queue-2 work; -> it covers `CR-16` and `CR-40`–`CR-42` there. Reviewer availability, not code, is this wave's critical path. +✅ **Merged to `main` on 2026-10-15 as [PR #94](https://github.com/HodeTech/Relavium/pull/94) (`ab08209e`).** +The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) joins +the forty item dispositions, the security sittings and the final evidence. Phase 2.5.5 exit criteria 1–3 and +sub-stream D are satisfied on `main`; M2.5.5-2's I half stays open. ### Wave 3 — Clear the ground -**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Steps 9a.1, 9a.2a, 9a.2b, NI, 9b.1 and 9b.2 accepted on `development` (2026-10-24/26), Steps 9c–19 NOT STARTED — 2026-10-26.** -ADR-0110–0113, their linked contracts and the nineteen-increment plan were approved on 2026-10-16. -[Steps 1–8 were independently accepted](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) -on `development`, with whole Step 8 reviewed at `c83c8575`, and published as the scoped checkpoint -[PR #95](https://github.com/HodeTech/Relavium/pull/95); its [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18) is the canonical record of -what merged. The phase register owns each increment's source, tests, cumulative review rounds, security sittings and -platform/publication evidence, including historical NONCLEAN R1–R24 records, the -[previous corrective review completion](phases/phase-2.5.5-hardening-and-remediation.md#pr-95-corrective-review-completion--2026-10-18), -the [second correction register](phases/phase-2.5.5-hardening-and-remediation.md#pr-95-second-systematic-review-corrections--2026-10-18), -the [second corrective source-review completion](phases/phase-2.5.5-hardening-and-remediation.md#pr-95-second-corrective-source-review-completion--2026-10-18) -(accepted at `ec0f3e10`), the [native Windows admission preparation](phases/phase-2.5.5-hardening-and-remediation.md#pr-95-native-windows-admission-preparation--2026-10-18) -and the [supplemental fixture acceptance](phases/phase-2.5.5-hardening-and-remediation.md#pr-95-supplemental-native-fixture-source-acceptance--2026-10-18). -Those scopes are not widened: the delta from `ec0f3e10` to the merged head is two independently accepted test-fixture -increments, documentation and the independent review's corrections at `a3e164b5`, which merged with author -verification only and were then independently reviewed after the merge -([review record](phases/phase-2.5.5-hardening-and-remediation.md#independent-review-of-a3e164b5-and-its-follow-up-corrections--2026-10-19)); -the follow-up corrections are on `development`, not yet on `main`. Whole-wave acceptance remains Step 19. No new decision, credential, billing or provider capture gate is introduced. - -Decompose, guard and normalize every hot file a 2.6 workstream is about to land on. -**The multi-file queue is Step 1 and runs first.** The approved -[nineteen-step execution plan](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments), -recorded as 29 increments since 2026-10-19, owns the full order and review gates: finish each increment and its fresh reviews -before advancing; no parallel edits of shared hot files. - -The groups below retain the original scope inventory and within-file dependency -arrows, not independent scheduling permission or current completion claims. The -[source-corrected preflight dispositions](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-preflight-and-proposed-execution-plan--2026-10-15) -own whether/how each original finding needs implementation or preservation. - -Two orderings here are not this plan's invention — they are the intra-phase serializations -[2.5.5 records under Positioning](phases/phase-2.5.5-hardening-and-remediation.md#positioning): **E before I** -on `home-controller.ts`/`chat-ink.tsx` (the decomposition moves already-guarded code), and **C before E** on -`packages/db/src/client.ts` (C types the error path in Wave 1; E's `openLocalDb` rewrite lands last, here). - -- ✅ **Multi-file Step 1 accepted — 2026-10-16** ([evidence](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-1-acceptance--2026-10-16)) — 2.5.5.E · `closeQuietly()` to its four siblings (#22) → `displayPath` - discipline to `resolve.ts`/`catalog.ts`/`chat-export.ts` (#4, G11) → `const exhaustive: never` across the - six sites (#300, #301, G3, which also makes `relavium run` render `agent:reasoning`). -- **`chat.ts` (strict)** — 2.5.5.G · the four money/token reimplementations (#236, #237, #242, #307) → - `swapAgentModel` → `reseatAgentModel` (#218) → 2.5.5.E · SIGINT actually calling `session.abort()` (#21, #23) - → `budgetWarningText` → **2.5.5.I · decompose `chat.ts`** (#220, #244). Then **2.6.M's `allowedDomains` - chat gap (G30)**, pulled forward into `session-host.ts`. -- **`home-controller.ts` (strict)** — 2.5.5.E · `selectGatePrompter` test (G37) → the swallowed palette - rejection (#281) → **2.5.5.I · decompose `createHomeController` + the overlay discriminated union** - (#215, #243) → 2.5.5.G · the `Capability`/`Port` vocabulary split (#217, #222). - Settle #125/D26 before decomposition: the approved preflight certifies the planned scope of the real - in-Home-chat cost path and retaining chat-only scope. If a new bare-Home cost - surface is selected, wire its explicit session target before decomposition. -- **`chat-ink.tsx`** — 1,906 lines at preflight (the original queue recorded 1,795), decomposed by **neither** phase doc. - **Recommendation: decline** a new decomposition (decision D18); take only the exhaustiveness guard and the - min-terminal-size guard (#63), and hand restructuring to 2.6.M's render-v2 in Wave 5a. -- **Viewport (strict)** — 2.5.5.I · share the ADR-0069 grapheme segmenter (#58, #62, G40, G51) → - 2.5.5.G · tab expansion before width measurement (#65) → per-cluster width cache + resize debounce - (#64, #66), refined to frame coalescing in the proposed plan. Alt-screen word-wrap - (#61) retains the approved Wave 5a ownership; it is not an unapproved Wave 3 prerequisite. -- **`packages/shared` (strict)** — the `.code` ruling (G50, D10) → `findDuplicates`/`superRefine` implementation or preservation at the - schema dispositions (#89, #90, G16, G17, G18, #210, #211, #212) → `BaseEventSchema` disposition (#213) → - the `RelaviumError` base class (#219). *The ruling must precede the base class, or it is baked in permanently.* -- **`registry.ts` / `builtins.ts` / `engine.ts`** — declared-source context repair and honest unsupported-mapping disposition (#92/Accepted ADR-0113) → `ctx.fsScope` (#93) → - `ok`/`success` (#216, #221) → identifier-sort `localeCompare` pinning (#258, #259) → the `git_commit` - denial copy (#308); then the four schema length caps (#95), the `llmVisibleParams` rewrite (#96, #98), - stale doc comments (#78, #97, #214); then `#armRunTimeout` (#75), `#dispatch` dead branch (#79), - `AbortSignal` into `resolvePrompt` + the human-gate recheck (#76, G9). -- **Process/CLI boundary (strict head)** — the revised plan serializes pure flag/descriptor/owner substrate → lazy guarded runtime graph → native crash integration → map/archive custody → command behaviour. It covers 2.5.5.E · the five crash/signal net gaps (#40, #41, #42, #224, - #269), the `--help`/`--version` historical ~400 ms tax (#43, #44, #271), and the flag-system ruling (#45, #264, - #265, #268), the exit-code and `--help --json` rulings (G35, #13), durable-only logs preservation (G15 refuted), the `--input` - namespace ruling (#18, #310). **`openLocalDb` wrap-and-redact (#278, #280) lands LAST** — it touches 8+ - call sites the other queues are still moving. -- The numbered plan assigns the remaining 2.5.5.A/E/G/I tails and keeps the central - DB refactor last before whole-wave acceptance. Step 19 owns sub-stream **A, E - and G sign-offs**, the reconciled **EXIT:4 reading** (D22), and **M2.5.5-3** - closure. Full I acceptance remains in Wave 5a. +**Paused on 2026-10-26 after Step 9b.2; 25 of 32 increments pending behind the MVP.** Steps 1–8 merged to `main` +as [PR #95](https://github.com/HodeTech/Relavium/pull/95) on 2026-10-18 ([merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-18-merge-checkpoint--2026-10-18)); +PR95, 9a.1, 9a.2a, 9a.2b, NI, 9b.1 and 9b.2 are accepted on `development`, each with its acceptance record in the +[register](phases/phase-2.5.5-hardening-and-remediation.md). The +[increment table](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) owns the +remaining increments' scope and order. Whole-wave acceptance stays Step 19. + +**Recorded splits moved into the MVP (D65, ID coverage unchanged):** row 10b's excess-argument refusal and +row 15a's #65 (MVP-2), row 15c's #63 (MVP-3), row 11b's Ctrl-C at a pending approval (MVP-4, now under the +keymap ADR), row 16b's #51 and row 17a's #24 (MVP-2), and G30's rows 13a–13c (MVP-7). Each keeps its recorded ADR +amendment and security sitting; the [priority reset record](phases/phase-2.5.5-hardening-and-remediation.md#mvp-priority-reset--2026-11-04) +records the within-row order amendments. 9c and 9d (with Wave 4b's production recipient) stay here under release +path B (D59), until 1.0 or until they land. + +Two serializations survive the pause: **E before I** on `home-controller.ts`/`chat-ink.tsx` and **C before E** on +`packages/db/src/client.ts`, as [2.5.5 records under Positioning](phases/phase-2.5.5-hardening-and-remediation.md#positioning). +Experience code from the MVP lands on the undecomposed `chat.ts` and `home-controller.ts`; Steps 12 and 14 rebase +over it when they resume (accepted cost, D65). ### Wave 4a — The spine @@ -1249,7 +247,8 @@ adapter/catalog money floor, cut the release 2.6.Q P6 is gated on. seeded-then-migrated harness #291; …) · 2.5.5.H's remainder (engine-deps allowlist #313/#318/#285; the docs-style checker #316; the logging-standard ruling #266/#267/#270; `fast-check` pilot #299) · 2.5.5.F's 20-site not-yet-built hedge sweep (#122 …) **rebased onto Wave 0's edits, not reverting them**. -- **Sub-stream B, C and H acceptance sign-offs**, then **cut and publish v0.2.0**. **Closes M2.5.5-4.** +- **Sub-stream B, C and H acceptance sign-offs. Closes M2.5.5-4.** *(Moved 2026-11-04: v0.2.0 is cut by the CLI + MVP's MVP-10, not here; M2.5.5-4's acceptance never contained it.)* ### Wave 5a — Paint before you build @@ -1330,7 +329,7 @@ unanswered**; the remainder sit inline in their own phase-file bullet. | D3 | 0 | Accept the upstream price changes the ADR-0071 §9 guard is refusing? | ✅ **Ruled + executed 2026-07-26: take current prices.** Verified: `gemini-flash-latest` moved $0.30→$1.50 in / $2.50→$9.00 out (the shipped floor under-priced by 5×) and `gemini-flash-lite-latest` $0.10→$0.25 / $0.40→$1.50. The nine upstream retirements were accepted in the same ruling; snapshot regenerated and merged in **PR #79**. Nothing outstanding | | D4 | 0 | Confirm the number reservation (`0013`–`0017`, ADR-0073+) | As listed — one item per number, in landing order | | D5 | 0 | The binding locale bar | ✅ **Ruled + propagated 2026-07-26: ship `en` + `tr`**, catalog architected for n locales, `es`/`fr`/`de` staged. Amended at all seven sites | -| D7 | 0 | Publish v0.1.1 as-is, or supersede with v0.2.0? | v0.2.0 — ADR-0067's Node `>=22` bump is breaking for 0.x | +| D7 | 0 | Publish v0.1.1 as-is, or supersede with v0.2.0? | v0.2.0 — ADR-0067's Node `>=22` bump is breaking for 0.x · **Refined 2026-11-04 by D58/D59** | | D8 | 1 | Do already-persisted approval previews need a scrub? | ✅ **Ruled 2026-07-30: NO scrub migration.** The recommendation had been "yes, migration 0013, same PR"; the ruling went the other way, so `0013` was left free by this ruling and is now taken by ADR-0074 §4's conservative money columns; the reservation ledger above shifts nothing — `0014` (#101) keeps its number. The forward fix shipped (#91: `previewFor` scrubs the display copy, `unredactedPreview` carries the in-process classification copy), so no NEW row can carry an unscrubbed preview; historical rows in a local `history.db` keep theirs. Recorded here because the shipped state contradicted this table for four days | | D10 | 1 | `BudgetExceededError`/`BudgetPauseError`: adopt `.code`? | ✅ **Ruled + executed 2026-07-30: adopt.** Both carry a stable `code` (`budget_exceeded` / `budget_paused`), with exported `isBudgetExceededError` / `isBudgetPauseError` guards — because `instanceof` is unsafe from a surface (the CLI bundles the engine, so two realizations of a class coexist and `instanceof` answers `false` at the boundary that must catch it). Same shape as `isCorruptRunEventError`. Precedes Wave 3's `RelaviumError` migration as required | | D12 | 2 | `~/.relavium/tmp/`: `tmp`-specific rule or the general home-anchored predicate? | General predicate accepted in [ADR-0105](../decisions/0105-filesystem-roots-are-invocation-grants-not-project-authority.md), with invocation authority and removal of unused tmp provisioning; approved 2026-10-12; implementation accepted in [Step 2](phases/phase-2.5.5-hardening-and-remediation.md#step-2-acceptance--2026-10-12). Full child/scratch lifecycle remains with Phase 2.6.N. | @@ -1348,13 +347,28 @@ unanswered**; the remainder sit inline in their own phase-file bullet. | D33 | 4b | Retention policy (the phase's largest single item) | Default bound 50 + `--limit`/`--tail`; ship `chat delete` writing the dead `deleted_at` columns; never auto-hard-delete | | D34 | 4b | Logging standard: build the injected-logger seam, or amend the standard? | **Amend** — building it adds a port to `packages/core`, needs its own ADR, breaks 2.5.5 exit criterion 8, and collides with 2.6.N's artifact-store port | | D37 | 4b | Provision CI provider keys for the live-nightly lane? | Yes, nightly-only, wired **before** the parity sweep | -| D50 | 6 | ADR row 11: adopt `ink-syntax-highlight`, or build in-house? | **Decline the dependency** per CLAUDE.md rule 2 — the markdown renderer is already in-house and untrimmed `highlight.js` is the phase's own named bundle risk | +| D50 | 6 | ADR row 11: adopt `ink-syntax-highlight`, or build in-house? | **Decline the dependency** per CLAUDE.md rule 2 — the markdown renderer is already in-house and untrimmed `highlight.js` is the phase's own named bundle risk · **Corrected 2026-11-04:** no markdown renderer exists yet; MVP-2 builds a minimal in-house one (ADR row 11, part 1) | | D52 | 6 | Draft ADR row 12 in Wave 6, not Wave 7 | Yes — settle the lineage event shape once, covering both 2.6.N and 2.6.P | | D57 | 7 | Dynamic runtime sub-workflow selection? | **Defer past the phase** — it is exactly the unauthored composition the *authored surface == executable surface* invariant exists to prevent | +| D58 | MVP | MVP scope | ✅ **Ruled 2026-11-04 — the larger MVP:** v0.2.0 on npm (milestone M3.5) after MVP-1…MVP-10, including conversational authoring, web, the Home hub and MCP/provider management; refines D7 | +| D59 | MVP | Release path under the interim guard | ✅ **Ruled 2026-11-04 — option B:** a superseding pre-1.0 release ADR publishes the smoke-verified package-test tarball from a trusted `v*` tag on `main` with a green `ci` at that commit; no archived encrypted maps for those releases; custody becomes mandatory again at 1.0 or when 9c/9d land. The guard stays in force until that ADR is Accepted and its guard change passes a security sitting | +| D60 | MVP | `LICENSE` §2 grants no usage right without a written agreement | ✅ **Ruled 2026-11-04 — unchanged.** Recorded risk for the published package | +| D61 | Gate 0 | `development`→`main` checkpoint before the MVP | ✅ **Ruled 2026-11-04 — yes,** a scoped checkpoint PR right after this reset; merging is the maintainer's action | +| D62 | MVP-3 | Default chat agent posture | ✅ **Ruled 2026-11-04:** read/search/find plus `write_file`/`edit_file` (hidden in ask/plan, per-call diff approval in accept-edits) plus `run_command` under D63; egress stays opt-in; the default mode stays `ask` with a `[chat].default_mode` setting and a one-time hint. Needs an ADR-0057 amendment and a security review | +| D63 | MVP-4 | `run_command` approval semantics | ✅ **Ruled 2026-11-04:** a typed card for any non-allowlisted command — once, session or remember; remembered grants live outside the repository in operator-owned config keyed by project root; accept-edits keeps prompting; headless stays fail-closed. New ADR (amends ADR-0029(a)/0061/0063) and a security sitting | +| D64 | MVP | Acceptance bar for MVP increments | ✅ **Ruled 2026-11-04 — tiered** (see [Next work](#next-work--the-cli-mvp)) | +| D65 | MVP | Paused Wave 3 increments' user-visible pieces | ✅ **Ruled 2026-11-04:** six recorded splits (10b, 15a, 15c, 11b, 16b, 17a) and #61 move into the MVP; Steps 12/14 stay paused | +| D66 | Gate 0 | `current.md`'s history | ✅ **Ruled 2026-11-04:** relocate the unique records, delete the duplicated history, keep every inbound anchor | +| D67 | MVP-4 | Context fidelity and "a user deny is final" | ✅ **Ruled 2026-11-04:** CR-71 un-deferred into MVP-4 (content-free marker), denials returned to the model, finite configurable ceilings; CR-70 with ADR-0096 §5 in v0.2.1 | +| D68 | MVP-1/2/4 | First-run defaults and the keymap | ✅ **Ruled 2026-11-04:** env-var key fallback, the interactive output budget, the default model following the stored key, a minimal `init`, and one keymap ADR (Ctrl+C interrupts the turn, a second press ends the session, Ctrl+C at a pending approval rejects it first) superseding the 2026-10-19 `/cancel` wording | +| D69 | MVP | Order of the added lanes | ✅ **Ruled 2026-11-04:** authoring → web → Home → MCP | +| D70 | MVP | npm pre-release before v0.2.0 | ✅ **Ruled 2026-11-04 — none** | +| D71 | MVP-5 | Parity items (caching, project instructions, type-ahead, `--model`, one-shot) | ✅ **Ruled 2026-11-04 — in the MVP** | +| D72 | — | The published `0.1.1`'s unsanitized `relavium run` output (#56) | ✅ **Ruled 2026-11-04 — no action;** the product has no users yet | ### Departures from the phase files' own sequencing -This ordering is compatible with both phase files' stated dependencies. It departs from their prose in six +This ordering is compatible with both phase files' stated dependencies. It departs from their prose in seven places, each deliberate: 1. **A baseline step neither file has.** Both treated `f88b0e8` as the working tree while it was not yet on @@ -1369,727 +383,38 @@ places, each deliberate: 5. **2.5.5's own eight exit criteria are certified nowhere in either file.** Criteria 1–3 are attached to Wave 2 and 4–8 to Wave 7; per-sub-stream acceptance sign-offs are attached to the last wave touching each. 6. **`chat-ink.tsx` is decomposed by neither file** despite being the largest TUI file. Raised as decision D18. +7. **The MVP reorder (2026-11-04)** puts Gate 0, MVP-1…MVP-10 and v0.2.1 ahead of the Wave 3 remainder and + Waves 4a–7, moves v0.2.0 out of Wave 4b (M2.5.5-4's acceptance never contained it) and pulls the items listed + under [Pending after the MVP](#pending-after-the-mvp-not-declined) forward. The waves' internal order is + unchanged. -## Where we are - -**Phase 1 — Engine and LLM is COMPLETE** (milestone **M2** reached, PR #27, 2026-06-16; -all workstreams 1.A–1.AH merged through PR #38, 2026-06-21). The engine runs end-to-end: -YAML parse → DAG → run loop → node execution (agent + the six non-agent handlers) → -checkpoint/resume → node retry → provider failover, with per-attempt cost tracking and a -gap-free `sequenceNumber`. The `@relavium/llm` seam is frozen and proven behind a -provider-agnostic `LLMProvider` interface with all three adapters (Anthropic, -OpenAI/DeepSeek, Gemini) and the `FallbackChain` runner. Two additive sub-spines also -completed: the agent-first entry point (`AgentSession` multi-turn + session persistence + -export-to-workflow, Lane C / 1.m5, 1.V–1.AA) and multimodal I/O (media input + engine -plumbing + inline & async output generation + generative adapters, 1.m6, 1.AD–1.AH). See -[Phase 1 detail](phases/phase-1-engine-and-llm.md), the [decision index](../decisions/), -and the [reference specs](../reference/). - -> **Repository checkpoint (recorded 2026-07-26; publication status corrected 2026-10-13):** -> (1) The CI `ci` job became a required check under `main` branch protection, alongside a -> `v*` tag ruleset limiting tag creation to maintain/admin. The optional `TURBO_TOKEN`/ -> `TURBO_TEAM` secrets for cross-runner remote cache were recorded as unset at that checkpoint; -> this does not assert their current configuration. (2) The former v0.1.1 publish-pending note -> was stale: the [npm registry](https://registry.npmjs.org/relavium) records `relavium@0.1.1` -> published on 2026-06-28. This resolves the package-publication status, without asserting the -> current `NPM_TOKEN`, npm 2FA or GitHub release-workflow state. The release procedure remains -> [ADR-0051](../decisions/0051-cli-distribution-thin-bundle-private-engine.md) / -> [release-a-surface.md](../runbooks/release-a-surface.md). - -## What is active now - -### 2026-07-19 full-project code review — triaged, Phase 2.5.5 opened - -A multi-agent code review of the whole tree at HEAD `f88b0e8` ran while the toolchain was green -throughout (lint, typecheck, and all 226 test files passing) — every finding is a defect a green -build cannot catch. 377 findings went through two-lens adversarial verification (one agent trying -to refute each finding against the code, one auditing its materiality); 373 survived and were -triaged into 205 work items. The review's central conclusion: the gap to first-class is a gap in -**propagation**, not competence — the correct pattern almost always already exists in the repo and -fails to reach the next site (a redaction helper not called twenty lines away, terminal -sanitization built for chat and never carried to `relavium run`, `withBusyRetry` applied to every -writer but the three oldest, cancel-wins implemented in four node handlers and omitted in the -fifth). That makes the remediation cheap and low-risk: the reference implementation is already -in-tree. - -Disposition: 176 work items opened the new **[Phase 2.5.5 — Hardening and Remediation](phases/phase-2.5.5-hardening-and-remediation.md)** -(engine/LLM/persistence hardening, MCP/secrets, the CLI operator-safety net, TUI terminal safety, -docs accuracy, CI/test-infrastructure, and hygiene), running **concurrently with Phase 2.6, not -before it** (327 findings); 27 findings (16 work items) were folded directly into the charter of an -already-open 2.6 workstream; 16 findings (10 work items) were already tracked in -[deferred-tasks.md](deferred-tasks.md) and were re-confirmed rather than duplicated; 3 needed no -action per the finding's own argument. Severity of the 373 triaged findings: 3 critical, 48 high, 134 -medium, 155 low, 33 polish. - -The three critical findings, visible here so they aren't buried in the phase file: - -- **A tool-approval preview that is not secret-redacted** (`packages/core/src/tools/registry.ts`) - — the persisted, `--json`-visible `ToolActionPreview` echoes a command/path verbatim instead of - going through the same redaction its sibling input-sanitizer already applies. -- **A transient `history.db` write failure silently drops a chat turn, then crashes the session** - (`apps/cli/src/chat/persister.ts`) — an async write failure surfaces after the reply was already - shown to the user, as an unhandled rejection. -- **Missing terminal-output sanitization on the `relavium run` TUI** (`apps/cli/src/render/tui/RunApp.tsx`) - — `stripTerminalControls`/`sanitizeInline` is applied at nearly every other dynamic-text display - boundary but never reaches this one, leaving an ANSI/OSC/Trojan-Source injection hole. - -### Phase 2.6.F — platform floor + the full-screen TUI renderer (merged to `main`, PR #74, 2026-07-11) - -The first 2.6 workstream. Behind [ADR-0067](../decisions/0067-node-supported-floor-22-reaffirm-better-sqlite3.md) -(Node `>=22` published floor, `>=22.13.0` dev-install floor; `better-sqlite3` re-affirmed), -[ADR-0068](../decisions/0068-full-screen-tui-renderer-ink7-harness.md) (`ink` 7 + a hand-built alternate-screen -renderer + the `ink-testing-library` harness), and -[ADR-0069](../decisions/0069-string-width-for-the-cli-renderer.md) (`string-width` for display width — **Accepted** -on the PR #74 merge, the CLAUDE.md rule-2 approval it gated). - -Shipped: `ink` 6→7 and the component-render harness; the alt-screen lifecycle with restore nets on every -termination path; a hand-built transcript **viewport** (grapheme-aware wrapping, windowing, scroll + auto-follow, a -per-entry wrap cache) and the inter-session alt-buffer **hoist**; the **`/scrollback`**, **`/edit`** and **`/copy`** -copy-and-search hatches over a `suspendFullScreen` primitive; **in-app mouse text selection with copy-on-select over -OSC 52** on both surfaces, with edge auto-scroll, a frozen auto-follow and a transient **`✓ Copied` toast** so the -otherwise-silent copy is confirmed; `--no-mouse` / -`[preferences].{alt_screen,mouse,copy_on_select,show_banner}`; a branded Home banner; and DEC-2026 synchronized -output — which `ink` 7 turned out to already emit, so the step **pins** it rather than building the planned -`stdout` Proxy. - -The full-screen renderer is now the **default on a TTY**; `--no-alt-screen` and a machine / non-TTY / `--json` / CI -path stay on the byte-identical inline renderer. - -Two adversarial review rounds (one per step, one over the whole phase) drove the last third of the work. The -whole-phase round found that **the caps-lift ADR-0068 exists to deliver was never implemented** — a name collision -with the Step-4b-3 wrap cache — so a >4000-character answer was still clipped in the very viewport built to scroll -it; that `displayWidth` under-counted 8 539 code points against `ink`'s own width function; and that a keyboard -Ctrl-C during a Home hatch stranded mouse reporting on the user's shell. All folded, each with a break-verified -regression test. - -### Phase 2.6.C — reseat transcript-carry + the `/cost` per-model breakdown (merged to `main`, PR #75, 2026-07-13) - -Driven by two maintainer manual-test findings. **F1:** a mid-session `/models` reseat **blanked the alt-screen -viewport** — the reseat builds a fresh view store, and on the full-screen renderer (the TTY default since 2.6.F, with -no native scrollback behind it) that store *is* the scrollback, so the whole conversation vanished the moment a user -switched models. The durable transcript and the new model's context were never at risk; the loss was the view. The -reseat now carries the rendered conversation, gated to full-screen (a seeded **inline** store would re-print the -conversation over the copy ink's `` already put in the terminal's scrollback), and the switch notice lands -**beneath** it as an inline `⇄ model changed → ` marker, keeping ADR-0059's bound disclosure clause. - -**`/cost`** now shows where the money went, behind a new -[ADR-0070](../decisions/0070-durable-per-model-session-cost-attribution.md): a durable `session_costs` table with the -invariant **`SUM(rows) == agent_sessions.total_cost_microcents`**, true by construction. ADR-0059 had planned to -derive the breakdown from the per-message `model_id`, which **cannot work** — one row holds one model, but a turn -whose tool loop fails over bills two. Along the way it closed a **live bug**: `total_cost_microcents` was being SET -blindly by five writers, so two `chat-resume` processes could permanently corrupt a session's total. It now has a -single owner. `session_messages`' never-written token/cost columns were dropped rather than left as a second, wrong, -zero-valued source. - -Four adversarial review rounds ran over the work and each found something real — including that a tripwire I shipped -was **armed in production** and **swallowed by ink** (so it could never fail), that a perf harness measured a -**pointer assignment** while its ADR note claimed it verified an O(n) bound, and that the `/cost` DB read — the -step's central change — had **zero test coverage** at any layer. All folded, each with a break-verified regression. - -**Open obligations carried out of 2.6.C:** - -- **`chat-resume` opens on an empty viewport** — it has never repainted prior turns, in any mode or version (nothing - projects `session_messages` into rendered entries). Not a 2.6.F regression; tracked in - [deferred-tasks.md](deferred-tasks.md). -- **Reported spend is a systematic under-estimate of the provider's invoice** — an egress that streamed content but - ended without a usage chunk, and a mid-stream failure, are recorded as 0 on *both* sides of the invariant. That is - a usage-capture gap in the seam, filed against **2.6.Q** (ADR-0070 §3). -- **Cross-turn tool-call memory** — a "the reseat forgets tool calls" report turned out to be an *engine* property, - not a reseat one: tool pairs never cross **any** turn boundary (ADR-0062 §6). Deferred behind a default-off toggle, - but **three already-live bugs gate it** (no pre-egress context-window check; overflow is a fatal, non-retryable - `bad_request`; the budget gate prices only output tokens) — see [deferred-tasks.md](deferred-tasks.md). -- **The CLI e2e suite migrates the developer's real `~/.relavium/history.db`** — verified, and it is what let a - broken migration reach real data while staying invisible to CI. Tracked in [deferred-tasks.md](deferred-tasks.md). - -**Open obligations carried out of 2.6.F:** - -- **The Home landing can overflow a short terminal** — a populated "Attention required" section pushes the top off, - and the alt buffer has no scrollback to recover it. Predates 2.6.F (the strip is 2.5.B); **2.6.G**'s management - browsers replace the strip wholesale, which is the right place to fix it. -- **`relavium run`'s TUI stays inline** (a deliberate 2.6.F scope cut); its retained-scrollable-history is tracked. -- **Themes** (beyond colour / `NO_COLOR`) are deferred to **2.6.L**, per the maintainer's Step-5 decision. -- Real-TTY signal paths (double-Ctrl-C, `kill -TERM`/`-HUP`/`-QUIT`, a hatch under `tmux`) remain a **manual PR-time - check** — the unit tests pin the orchestration around injected seams, not the kernel. - -**Phase 2 — CLI (milestone M3) is feature-complete** (every in-phase workstream 2.A–2.S merged; published as -**v0.1.1**). The CLI is the first real -`@relavium/core` consumer and doubles as the engine's regression harness — validating the -engine API ergonomics before the desktop and VS Code surfaces. **Landed:** the CLI skeleton + -process contract (**2.A**) and the two-level config-resolution loader (**2.B**), both ✅ Done -(PR #40, 2026-06-22) behind [ADR-0047](../decisions/0047-cli-framework-commander-ink-clack.md) -(commander/ink/@clack) and [ADR-0048](../decisions/0048-toml-config-parser.md) (smol-toml); and -**2.D** (`relavium run` wired to `@relavium/core` — the M3 keystone and first real engine -consumer), ✅ Done (PR #41, 2026-06-22), which also adds the `defaultProviders()` seam registry; -and **2.F** (the `--json` CI machine-output contract — pure-NDJSON stdout, diagnostics → stderr), -✅ Done (PR #42, 2026-06-22) behind [ADR-0049](../decisions/0049-cli-machine-output-contract.md); -and **2.K** (the engine regression harness, now the engine's CI regression gate), ✅ Done -(PR #43, 2026-06-23) — **reaching global milestone M3**; and **2.H** (durable local run history via -`@relavium/db` — the `RunStore` writer + read API the gate-resume/list/logs/status surfaces consume), -✅ Done (PR #44, 2026-06-23) behind [ADR-0050](../decisions/0050-cli-history-db-at-rest-posture.md); -and **2.C** (the `relavium provider` commands — a provider registry + API keys in the OS keychain via -`@napi-rs/keyring`, resolved keychain → `RELAVIUM__API_KEY` env var → error), ✅ Done -(PR #45, 2026-06-23) behind [ADR-0019](../decisions/0019-cli-node-keychain-library.md) + -[ADR-0006](../decisions/0006-os-keychain-for-api-keys.md) (no new ADR — `secrets.enc` deferred past v1.0); -and **2.E** (the `ink` streaming TUI — the third `RunRenderer` over the one event bus: live per-node status -+ spinners, the active node's streaming tokens, a running cost footer, a persistent final summary; cooperative -Ctrl-C cancel), ✅ Done (PR #46, 2026-06-24) behind [ADR-0047](../decisions/0047-cli-framework-commander-ink-clack.md) -(ink + React 19, confined to `apps/cli`; no new ADR); -and **2.G** (the interactive human-gate prompt — a `@clack/prompts` card during `run` — plus the out-of-band -`relavium gate ` cross-process resume: reload snapshot → reconstruct checkpoint → `resumeFromCheckpoint`, -idempotent, secret-input fail-closed), ✅ Done (PR #47, 2026-06-24) behind -[ADR-0047](../decisions/0047-cli-framework-commander-ink-clack.md) (`@clack/prompts`; Node floor 20.11→20.12; -no new ADR) — **fully closing 2.K's deferred gate-resume half**; -and **2.I** (the read commands `list` / `logs` / `status` / `gate list` over durable history — go/no-go #2, the -read side; surfaces the pending `gateId`s the 2.G `gate` command points at), ✅ Done (PR #48, 2026-06-24) -(no new ADR — an additive workflow-agnostic `@relavium/db` read seam + a `@relavium/core` `parseAgent`); -and **2.L** (packaging, distribution & install verification — go/no-go #7, the last gate-closing spine PR: the -`tsup` engine-inlined ESM bundle, the bundle-closure drift guard, and the tag-triggered cross-OS install-smoke -`Release CLI` workflow), ✅ Done (PR #49, 2026-06-24) behind -[ADR-0051](../decisions/0051-cli-distribution-thin-bundle-private-engine.md) — **closing go/no-go #7, so the -Phase-2 spine is complete and all seven Phase-3 exit criteria now hold (Phase 3 may start)**. -**Also landed — the first additive lane:** **2.S** (media host-wiring — the surface half of the multimodal -sub-spine: the `model_catalog` reader → `resolveMediaSurface` routing + the D15 catalog load-check shared by -`run`/`gate`, the content-addressed `MediaStore` de-inline to a `media://` handle, the SSRF-validated -`EgressCapability.fetch` egress, the containment-checked `save_to` write port, durable fail-cost on the terminal -events, the produced-media render surface, and the best-effort run-end host media GC), ✅ Done (PR #52, 2026-06-25) -behind [ADR-0042](../decisions/0042-engine-media-storage-substrate-mediastore-deinline-retention.md)–[ADR-0046](../decisions/0046-inline-media-out-via-generate-streaming-triad-deferred.md) -(no new ADR). -**Also landed — the first user-facing `AgentSession` surface:** **2.M** (`relavium chat` — the agent-first -interactive REPL over `AgentSession`: streaming tokens, tool-call annotations, the FS-scope tier + `allowedCommands` -allowlist honored, `git_commit` denied; `/exit` / `/cancel` / an input-stream EOF / raw-mode Ctrl-C all end the -session with **exit code 4** — over ONE framework-free command core driving both an `ink` TTY app and a plain -non-TTY line loop; a built-in default agent over `[chat].default_model` for a zero-config first run; durable -per-turn persistence to the shared `history.db` that round-trips via `reconstructSessionState`; the ADR-0028 -cost cap wired; model output + pasted input sanitized of terminal control sequences at the display boundary), -✅ Done (PR #54, 2026-06-26) — **no new ADR** (covered by [ADR-0024](../decisions/0024-agent-first-entry-point-agentsession.md), -[ADR-0047](../decisions/0047-cli-framework-commander-ink-clack.md), [ADR-0028](../decisions/0028-workflow-resource-governance.md), -[ADR-0050](../decisions/0050-cli-history-db-at-rest-posture.md), [ADR-0029](../decisions/0029-tool-policy-hardening.md)). -`read_media` **input** access (D12) — which 2.S had pointed at 2.M — was **split into a dedicated, -security-reviewed follow-up** (maintainer-approved); the 2.M REPL shipped without it (tracked in -[deferred-tasks.md](deferred-tasks.md)). -**Also landed — the rest of the agent-first chat family:** **2.N** (`relavium chat-resume` — reload + continue a -persisted session over a shared REPL), **2.O** (`chat-list` — over a new additive `SessionStore.listSessions` -read seam), **2.P** (`chat-export` + the in-REPL `/export` — session → `.relavium.yaml` scaffold, [ADR-0026](../decisions/0026-session-export-to-workflow.md)), -and **2.Q** (`chat --json` — a headless `SessionEvent` NDJSON driver — + the one-shot `relavium agent run` with a -minimal in-house `--fixture` cassette for deterministic offline replay), all ✅ **Done (PR #55, 2026-06-26)** — -**no new ADR** — completing the agent-first CLI lane. (`agent run --input` is reserved/rejected until session -`{{ctx.*}}` prompt interpolation lands — a tracked engine follow-up in [deferred-tasks.md](deferred-tasks.md).) -**Also landed — 2.R (the inbound MCP client):** agents now consume external MCP servers' tools across `chat`, -`run`, and one-shot `agent run`. The **`@relavium/mcp`** foundation (the SDK-fenced package, the dependency-free -JSON-Schema→Zod compiler, the fail-loud connect-all manager, the `mcp_{server}_{tool}` namespacing) is ✅ **Done -(PR #56, 2026-06-26)**, and the **host wiring** (chat/run/agent-run), the **network transports** (`http`/`sse`/ -`websocket`) behind the SSRF pre-connect floor + the per-server `allow_local_endpoint` opt-in, **named secrets** -via the isolated `mcp-secret:*` keychain namespace, the by-name `ref` registration form, and the **real-spawn -e2e** are ✅ **Done (PR #57, 2026-06-27)** — behind [ADR-0034](../decisions/0034-mcp-client-sdk-dependency.md), -[ADR-0052](../decisions/0052-inbound-mcp-client-package-lifecycle-registration.md), and -[ADR-0053](../decisions/0053-mcp-network-transport-egress-security.md). It was off the M3 critical path and the -Phase-3 go/no-go (capability without gating). Residual MCP hardening — the connect-by-validated-IP dialer, -network header-auth, tool-list caching, mid-call abort propagation — is tracked in -[deferred-tasks.md](deferred-tasks.md). The **stdio spawn is gated** as of 2.6.5 `CR-16` -([ADR-0084](../decisions/0084-consent-before-a-local-mcp-spawn.md)): consent per machine and per resolved -declaration, on every path that can open an MCP-bearing artifact. The "import-trust" framing is retired — a -`git pull` changes a committed artifact with no import step — and only the `npx` version/integrity pinning half -remains deferred. -**Also landed — 2.J (the YAML-authoring lifecycle), the last in-phase lane:** `relavium create` (a -`@clack/prompts` wizard scaffolding an agent **or** a minimal single-agent workflow, validated against the -kind-appropriate `@relavium/shared` schema before write, dual-TTY-gated), `relavium import ` (schema- -validated copy-in with **project-global** id uniqueness), and `relavium export ` (a portable copy -**re-serialized from the validated AST** — canonical, comment-free, no provider key by construction), sharing one -`assertSlugAvailable` cross-catalog guard, ✅ **Done (PR #58, 2026-06-28)** — **no new ADR** (covered by -[ADR-0026](../decisions/0026-session-export-to-workflow.md)/[ADR-0047](../decisions/0047-cli-framework-commander-ink-clack.md)). - -**Phase 2 — CLI is feature-complete.** Every in-phase workstream (2.A–2.S) is merged and the published CLI is cut -as **v0.1.1**; M3 was reached at 2.K and the Phase-3 go/no-go held from 2.L. See the -[Phase 2 workstreams](phases/phase-2-cli.md) and the -[sequencing plan](phases/phase-2-cli.md#sequencing--parallelization). The full status-aware history is the -[Remaining build order](phases/phase-2-cli.md#remaining-build-order) section (its queue is now empty). - -**Phase 2.5 — CLI Consolidation & Conversational Home has started.** **2.5.A** (the spine's secure base — a -shared `assembleToolEnv({ profile, fsScopeTier, workspaceDir })` factory wired into **both** the chat and -workflow-run paths, the host-side `fs` (`realpath`+`commonpath` jail, read-only chat fail-close, single-fd -`O_NOFOLLOW`/`O_NONBLOCK` reads) and `process` arms, the advertise-filter, **EA1** `tool_unavailable`, and **EA2** -real failed-turn usage) is ✅ **Done (PR #60, 2026-06-28)**, behind -[ADR-0055](../decisions/0055-cli-host-capability-seam-tool-environment-factory.md) — **reaching milestone -M2.5-1 (secure base)**. The `egress`/`os` arms, the `project`-tier `extraRoots` allowlist, and a write-capable -chat are deferred to **2.5.E**/[ADR-0057](../decisions/0057-cli-chat-modes-and-per-tool-approval.md) (tracked in -[deferred-tasks.md](deferred-tasks.md)). **2.5.B** (the bare-invocation Home) is ✅ **Done -(PR #61, 2026-06-29)**, behind [ADR-0054](../decisions/0054-cli-bare-invocation-interactive-home.md) (Accepted): the -TTY-gated bare `relavium` opens a read-only management strip (recent sessions/runs/agents + an "Attention -required" section of pending human gates / failed runs) over a bounded, indexed `history.db` read seam, sitting -above a live prompt that graduates into an in-process chat; rendered as a single ink tree (one `useInput` owner) -with one SIGINT/SIGTERM lifecycle (clean Home exit `0`; an external signal → the conventional `128+signo`, 130/143; -the in-Home chat's exit-`4` consumed, never leaked) and bracketed paste (DECSET 2004), all while every -non-interactive path keeps the byte-for-byte help + exit-`0` meta-op ([ADR-0049](../decisions/0049-cli-machine-output-contract.md)). -Canonically homed in [home.md](../reference/cli/home.md). **2.5.C** (the in-app command system) is ✅ **Done -(PR #62, 2026-06-30)**, behind [ADR-0056](../decisions/0056-cli-in-app-slash-command-system-and-manifest.md) -(Accepted): a curated **two-registry** model (the shell `COMMAND_MANIFEST` driving `commander` + `--help --json` -+ the `executeCommand` dispatch, vs the in-REPL `REPL_COMMANDS` driving a filterable `/` palette + slash commands -in **both** the chat and the bare Home — no command in both); `/help`, the `notice` output channel, `/workflows`, -`/cost`, and `/doctor` (fast tier: keychain/config/wired-tools; `--deep`: a **redacted** provider-key probe + a -**read-only** MCP-status report — a security-review decision: it reports the live session's already-connected -servers, never a fresh connect/spawn); plus the `name + args` slash dispatch and a context-aware footer hint-bar -surfacing `/ for commands`. Canonically homed in [commands.md](../reference/cli/commands.md) + -[chat-session.md](../reference/cli/chat-session.md). **2.5.E** (chat modes + per-tool approval + mid-turn abort) -is ✅ **Done (PR #63, 2026-07-03)**, behind -[ADR-0057](../decisions/0057-cli-chat-modes-and-per-tool-approval.md) (**Accepted** after the mandatory -security review): the reseat-less mode system (ask / plan / accept-edits / auto on `Shift+Tab` + `/mode`), the -fail-closed per-tool `confirmAction` floor (`[y]/[a]/[n]` + a session once/always cache), the `Esc` mid-turn -abort (EA7), and the host arms closing the 2.5.A deferral — a write-capable `fs` tier + **protected paths** -(refused in every mode incl. `auto`), the SSRF-hardened `egress` arm (shared with media), and the `os` arm (now -a governed action class). Wired live into `relavium chat`, one-shot `agent run`, and the Home (each activates -the regime before its first turn). Every step passed the mandated opus + Sonnet 5 loop plus the dedicated -holistic security review (~50 findings fixed, 4 HIGH). A same-PR chat-UX follow-up also landed: a host tool -EXECUTION failure on the interactive surface (a file-not-found READ) is fed back to the model to recover -(`recoverToolFailures`, scoped to IDEMPOTENT tools via a stamped `ToolExecutionError.recoverable`; a governed / -side-effecting failure stays fail-fast) plus a static secret-free `tool_failed` hint. **With 2.5.E the CLI -Consolidation spine (2.5.A → C, E) is complete.** **2.5.D** (chat input ergonomics) is ✅ **Done (PR #64, merged -2026-07-03)** — the first experience-arm workstream: the pure-ergonomics half (`Ctrl+J` multiline, -`↑/↓` history + `Ctrl+R` reverse-search, readline motions, a shared cursor-bearing `EditorState`) plus the two -data-moving affordances behind [ADR-0061](../decisions/0061-cli-input-layer-file-injection-and-shell-escape.md) -(**Accepted** after a two-round maintainer security review): **`@`-mention** (dir-navigable file completion that -reads through the SAME `FsCapability` `read_file` uses — jail + the expanded sensitive-read confidentiality floor -+ listing-gate + binary/size guards — and injects as UNTRUSTED, nonce-fenced, byte+line-bounded context) and -**`!`-shell** (the additive `AgentSession.runUserCommand` — EA8 — routing `!command` through the one `run_command` -boundary: `enforcePolicy([chat].allowed_commands)` BEFORE the mode-aware `confirmAction` → `spawn`/`shell:false`; -**empty-default allowlist ⇒ `!` inert**, secure-by-default, with an actionable deny hint). Each of the 5 steps -passed the mandated opus + sonnet adversarial-review loop, and the ADR-0061 mandatory security pass ran inside the -step-4/5 loops (14 findings fixed across the two `@`-mention rounds; on the `!`-shell the opus + security pass -confirmed 0 defects after adversarial verification, and the sonnet second pass caught a HIGH — a `!`-command in -flight left no host-visible busy signal, so a message typed mid-command could crash the session — now fixed with a -`shellBusy` input gate, plus a LOW type-hygiene fix). A post-implementation comprehensive review then refined the -`@`/`!` presentation to a **pending-attachment (chip) model** (an inline `@path` marker + a read-only `!`-output -preview, expanded into the SAME UNTRUSTED nonce-fenced frame only at submit — byte-identical model context, a -clean prompt), and two follow-up review passes hardened the `[chat]` allowlist resolution (exact + glob arrays are -now a **coupled unit**) and fixed a Backspace regression (ink reports the Unix physical Backspace as `key.delete`); -all recorded in the ADR-0061 "Refined at implementation" append. **2.5.F** (the ADR-0062 context commands) is ✅ -**Done (PR #65, merged 2026-07-05)** behind [ADR-0062](../decisions/0062-context-compaction-and-cli-history-commands.md): -`/clear` (a host-level fresh-session swap across `chat`/`chat-resume`/Home, TTY-interactive only), the -`session:compacting` "Summarizing…" moment event (amending ADR-0036), and the footer context-fullness indicator — -completing compaction alongside the earlier `/compact` + `/trim` + automatic compaction. **2.5.G** (onboarding -wizard + Home `/models` + the live model catalog) is ✅ **Done (PR #66, 2026-07-07)** — its scope **expanded to -Option A**: a **live** model catalog (per-key provider discovery + a DB cache + refresh + a static/live merge) -and a complete model-pricing story (user-supplied pricing that governs cost), behind three ADRs -([ADR-0063](../decisions/0063-cli-config-write-contract.md) config-write · -[ADR-0064](../decisions/0064-live-model-catalog.md) live catalog · -[ADR-0065](../decisions/0065-provider-economics-and-extensibility.md) provider economics), across 12 reviewed -steps (all landed); with it **milestone M2.5-2** is reached. The additive lane **2.5.H** (reasoning render + live-turn feedback + an actionable error taxonomy) is ✅ -**Done (PR #67, 2026-07-07)** — behind **EA6** (a new dual-envelope `agent:reasoning` stream event that *amends* -[ADR-0036](../decisions/0036-run-loop-substrate-event-bus-and-execution-host.md); no new top-level ADR): a -host-emit of the reasoning the `@relavium/llm` seam already carries (ADR-0030), a collapsible "thinking" panel -(`/thinking` / `Ctrl+T`), the `Thinking…/Working… {elapsed}s · Esc to stop` timer, a visible `…` elision marker -(fixing the silent live-buffer head-drop), per-attempt `via {model}` failover attribution, and a static, -secret-free per-`ErrorCode` recovery hint (session-survives; a context-overflow message heuristic → `/compact`· -`/trim`). Four steps, each opus + Sonnet reviewed (3 HIGH fixed: a run-path silent-drop, a frozen Home timer, a -scrollback elision loss; + a one-shot `agent run` hint-leak). **With 2.5.E this reaches milestone M2.5-3.** The -consolidation lanes **2.5.I** (regression harness + DB concurrency hardening — `loadFull` read-txn snapshot, -`BEGIN IMMEDIATE` writes with a deterministic `SQLITE_BUSY` retry, the concurrent chat+run + cassette-chain + -perf-budget e2es, an advisory Windows CI lane) and **2.5.J** (docs-debt: the accurate unencrypted-history -posture per ADR-0050, and `NO_COLOR`/`FORCE_COLOR`/`--color` resolution) are ✅ **Done (2.5-close-out, PR #69, -2026-07-08)** — **reaching milestone M2.5-4, so Phase 2.5 is complete, merged to `main`** — landed alongside the doable-now -Batch A–E backlog (test-hardening; 2.5.H TUI polish; `AgentParseError` line/col; the ADR-0057 approval/security -batch — `[c]` reject-with-reason, non-TTY policy, SCOPE-denial recovery, Ctrl+T-in-approval, the Trojan-Source -bidi floor, behind an append-only ADR-0057 amendment; the profile-aware advertise-filter + the in-house -`.gitignore` matcher), each implement → Opus → Sonnet with a security-review pass on the approval batch. Two -`gate.ts`-resume items (`relavium budget resume` + secret re-provide) are deferred to a focused follow-up, and -the session `{{ctx.*}}` interpolation stays with the Proposed ADR-0060 (Phase-2.6); all tracked in -[deferred-tasks.md](deferred-tasks.md). See the -[Phase 2.5 workstreams](phases/phase-2.5-cli-consolidation.md). A **post-2.5.G model-UX follow-up** (from six -maintainer questions on model/provider/wizard behavior) then landed as a sequenced plan: `/models` key-awareness, -the onboarding-wizard live key-validation + retry UX, wizard-dynamic provider docs, **mid-session model switching** -(the `/models` reseat across `relavium chat` + the in-Home chat — [ADR-0059](../decisions/0059-cli-mid-session-model-reseat.md), -now Accepted), and **normalized reasoning-effort control** ([ADR-0066](../decisions/0066-normalized-reasoning-effort-control.md), -now Accepted, **merged 2026-07-07**): a provider-agnostic effort tier (`off`/`low`/`medium`/`high`/`max`) authored in -agent YAML or the `[chat].reasoning_effort` config default, each adapter mapping it to its provider's **native** tier — -**all four providers**, DeepSeek-v4's `thinking` param included (doc-verified against api-docs.deepseek.com) — gated -per-model by a host-injected capability resolver (plus a conservative id heuristic for a live-discovered model). It is -changed **interactively**: `/effort` opens a keyboard-owning tier-selector **overlay** in `relavium chat` + the in-Home -chat (§6), and the `/models` picker's **effort sub-step** either applies a **per-turn session override** in a live chat -(no reseat, §5) or — in the **bare Home** — **writes** the model + effort as the next session's config defaults (a new -global `[preferences].reasoning_effort` key), the active tier shown in the footer. Two multi-agent review passes -hardened it: the first caught the picker (wrongly) routing effort through a full model reseat — the P0 fix rebuilt it -as the ADR-mandated session-level setter; the second caught the bare-Home effort default not being re-read for a -same-process next chat — fixed to mirror the model's fresh per-chat re-read. - -Carry-over hardening is tracked in [deferred-tasks.md](deferred-tasks.md) — Phase 2 picks -items up as it first touches each file. Notable inheritances: 1.AH's host-wiring half -(distributed across Phases 2–6), the media-egress host-side SSRF mechanism, and the -keychain no-raw-key IPC test. - -## Not started yet - -The rest of **Phase 2.6 — Conversational Authoring and the First-Class CLI** -([phase-2.6-conversational-authoring.md](phases/phase-2.6-conversational-authoring.md), in progress — -**2.6.F is done**, -unblocked by the 2.5 close and **re-scoped 2026-07-08** from maintainer UX findings + a competitor -research pass + the deferred-tasks triage): a full-screen, Home-managed CLI (browsers for -workflows/runs/agents, provider + MCP + settings management, onboarding v2 with the Relavium-account -stub), competitor-breadth tools under the governance floor, settings/theming/`en`+`tr` localization, -the run-ops resume follow-up — plus the original spine: the shared `@relavium/authoring` package and -a chat that **authors** standards-valid `.relavium.yaml`. Then the surfaces and the cloud — -everything after the engine critical path: the desktop app (Phase 3) and the VS Code extension (Phase 4), then **Product Phase 2** — first **managed -inference** ([phase-5-managed-inference.md](phases/phase-5-managed-inference.md), the opt-in -`managed` gateway, engine still local), then the **cloud execution layer and web portal** -([phase-6-cloud-execution-portal.md](phases/phase-6-cloud-execution-portal.md)), the two -decoupled per Option B. See the [phase index](README.md#phase-index) and the -[milestone spine](README.md#global-milestone-spine) (M3 onward). - -**Systematic group 2a, round 3 — 2026-10-04.** [Two complete independent reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T15-44-35-w7-systematic-group-2a-round-3-review.md) -verify all four tool notifications, then reproduce an inherited raw-callback usage/hard-cap gap and -skipped realized recording. `e7973031` preserves original error identity while carrying canonical -accounting and starts mandatory recording after authoritative cost aggregation despite delivery failure. -Original baseline: 40 failing/13 passing; corrected root: 7,625 passing/12 skipped across 331 files, -23 forced tasks and six forced builds. Every sealed report/entry is verified before Original resumes. -**Fresh complete round 4 is required; group 2a, ownership, remaining groups, Steps 7–8/12, Sonar and PR remain open.** - -**Systematic group 2a, round 4 — 2026-10-04.** [Both fresh complete reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T16-34-57-w7-systematic-group-2a-round-4-review.md) -reproduce frozen classified errors defeating the carrier, proven pre-provider refusals consuming -slots without a governor, and classified success-flush errors losing usage or counting twice. -`ecf0b209` separates canonical accounting from throwable mutation/classification and records -actual chain-owned provider invocation; untyped chain diagnostics are fixed. Original baseline: -34 failing/243 passing; corrected focused run: 451 passing. Forced root checks pass all 23 tasks, -332 files, 7,665 tests/12 skips, six builds and test isolation. Both full reports and every one of -3,524 sealed entries/pins are independently verified before Original resumes. **Fresh complete -round 5 is required; group 2a, ownership, remaining groups, Steps 7–8/12, Sonar and PR remain open.** - -**Systematic group 2a, round 5 — 2026-10-04.** [Both fresh complete reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T20-43-04-w7-systematic-group-2a-round-5-review.md) -reproduce hostile error presentation replacing the original throwable/suppressing its terminal, -method lookup counted as invocation, and valid generated usage lost on pricing failure. -`902737eb` guards presentation, resolves receiver-bound methods before invocation and retains -safe-integer validated usage without inventing a cost. Original baseline: 19 failing/287 passing; -corrected focused run: 417 passing. Final forced root checks pass 23 tasks, 332 files, -7,694 tests/12 skips, six builds and test isolation. Both complete reports and every one of -3,589 sealed entries/pins are independently verified before Original resumes. **Fresh complete -round 6 is required; group 2a, ownership, remaining groups, Steps 7–8/12, Sonar and PR remain open.** - -**Systematic group 2a, round 6 — 2026-10-04.** [Both fresh complete reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T21-48-21-w7-systematic-group-2a-round-6-review.md) -verify existing protections and reproduce six overlapping price/quantity/invocation/diagnostic families. -`b4134e99` owns validated returned usage, marks unknown prices unpriced, guards nested/cause inspection, -keeps admission ownership through discharge and rechecks lookup cancellation. Exact-head baseline: -25 failing/19 passing; corrected focus: 461 passing. Final forced root passes 23 tasks, 333 files, -7,738 tests/12 skips, six builds, formatting and test isolation. Both complete reports and all -3,922 sealed entries/source/link/bootstrap pins are independently verified before Original resumes. -**Fresh complete round 7 is required; group 2a, ownership, remaining groups, Steps 7–8/12, Sonar and PR remain open.** - -**Systematic group 2a, round 7 — 2026-10-04.** [The complete contracts review and incomplete runtime observations](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T23-06-22-w7-systematic-group-2a-round-7-review.md) -reproduce generated observer errors causing paid retries/private public diagnostics and second tool-round -reflection replacing callback failures. `dd9d5a7d` retains exact attempt-observer provenance and guards -budget inspection. Original permanent baseline: 7 failing/24 passing; corrected focus: 200 passing. -Final forced root passes 23 tasks, 335 files, 7,769 tests/12 skips, six builds and test isolation. -Both entire reports and all 4,140 sealed entries/source/link/bootstrap pins are independently verified -before Original resumes. Runtime's platform-interrupted review is explicitly incomplete, with no -restored runtime certification. **Fresh complete round 8 is required; group 2a, ADR-0102 approval, -remaining groups, Steps 7–8/12, Sonar and PR remain open.** - -**Systematic group 2a, round 8 — 2026-10-05.** [Both complete independent reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-05T00-56-00-w7-systematic-group-2a-round-8-review.md) -verify held terminal mutation before confirming EOF, mutable attempt diagnostics and typed paid -observer failures granting unintended workflow retry/budget authority. Corrections own the held -terminal, freeze diagnostic copies and carry exact external observer origin separately from internal -admission faults. Session classified delivery and actual money failure ownership remain intact. -The 51 new terminal controls reproduce 31 failures before repair; final forced Original checks pass -23 tasks, 336 files, 7,831 tests / 12 existing skips, six builds and test isolation. Both complete -reports and all 4,175 physical entries/pins/links are verified before Original resumes, with prior -outer-launcher gaps qualified. **Fresh complete round 9 is required; group 2a, ADR-0102 approval, -remaining systematic groups, Steps 7–8/12, current Sonar and PR remain open.** - -**Systematic group 2a, round 9 — 2026-10-05.** [Both complete independent reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-05T01-57-06-w7-systematic-group-2a-round-9-review.md) -verify unsafe cause authority, generated result access after paid observation, compaction/later-tool -callback origin and false money-writer attribution. Verified corrections `1b23ff82` / `22d9df14` and -119 additional permanent controls pass forced full lint/typecheck/test (23 tasks, 339 files, -7,950 tests / 12 existing skips), forced build and isolation. Both complete reports and all 3,784 -physical entries/source/link/bootstrap pins verify before Original resumes; actual-price/unpriced -and early startup limits remain explicit. **Fresh complete round 10 is required; group 2a, -ADR-0102 approval, remaining groups, Steps 7–8/12, Sonar, W7 and PR #90 remain open.** - -**Systematic group 2a, round 10 — 2026-10-05.** [Both complete independent reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-05T02-56-00-w7-systematic-group-2a-round-10-review.md) -verify lifecycle setup/provenance and nested generated-content authority gaps. Corrections -`639d1ace` / `c6749597` add 94 controls and pass forced full lint/typecheck/test (23 tasks, -343 files, 8,044 tests / 12 existing skips), six builds and isolation. Both complete reports and -all 3,741 actual physical entries/source/link/bootstrap pins are audited; Contracts' three post-audit -cache files and one directory discrepancy explicitly qualify its seal claim. **Fresh complete -round 11 is required; group 2a, ADR-0102 approval, remaining groups, Steps 7–8/12, Sonar, W7 and -PR #90 remain open.** - -**Systematic group 2a, round 11 — 2026-10-05.** [Both complete independent reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-05T07-03-00-w7-systematic-group-2a-round-11-review.md) -verify terminal cancellation after successful durability, custom controller reentrancy and deadline -cleanup losing known price or acquiring control authority. Scoped repairs preserve canonical usage, -EA7 late abort and genuine money controls; 101 new core controls plus three shared disposal controls -pass. Forced Original checks pass 23 tasks, 351 files, 8,148 tests / 12 existing skips, build six and -isolation. Both exact final seals are independently audited; early evidence gaps remain qualified. -**Fresh complete round 12 is required; group 2a, ADR-0102 approval, remaining groups, Steps 7–8/12, -Sonar, W7 and draft PR #90 remain open. No additional paid generation is authorised.** - -**Systematic group 2a, round 12 — 2026-10-06.** [Qualified recovery and corrections](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-06T16-02-49-w7-systematic-group-2a-round-12-review.md) -record two interrupted reviews without full-scope acceptance. Cold reconstructed compaction preserves -terminal cancellation; generated typed output is owned before host pricing while known quantities -remain chargeable on projection failure. Thirty-one controls pass; forced Original lint/typecheck/test -passes 23 tasks, 355 files, 8,179 tests / 12 existing skips, six builds and isolation. Both complete -recovery reports and all 3,534 physical entries are audited. **Fresh complete round 13 is required; -group 2a, remaining groups, Steps 7–8/12, Sonar, W7 and draft PR #90 remain open.** - -**ADR-0102 maintainer clarifications — 2026-10-06.** [Two fresh qualified static reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-06T16-02-49-w7-adr-0102-maintainer-clarifications-review.md) -verify all six submitted clarifications at `1f7c353e`; no new material draft finding. SDK HTTP acceptance -is still required during implementation. **ADR-0102 stays Proposed, pending maintainer approval. -No dependent ownership implementation or additional paid generation is authorized.** - -**Systematic group 2a, round 13 — 2026-10-06.** [Both complete independent reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-06T16-53-54-w7-systematic-group-2a-round-13-review.md) -reproduce a cold provider resolver starting a real nested turn, after which compaction overwrites -its controller. `9f35173a` rechecks existing idle ownership before arming compaction; cold cancel, -idle abort and warm memoization remain intact. The Original regression fails before correction; -final forced lint/typecheck/test passes 23 tasks, 357 files, 8,195 tests / 12 existing skips, -six build tasks and isolation. Both reports, the complete Runtime static supplement and all -3,526 physical entries are audited, with early launch/read/preparation gaps retained. -**Fresh complete round 14 is required; group 2a, Proposed ADR-0102 approval/implementation, -remaining groups, live-fixture-gated Steps 7–8, final Step 12, Sonar, W7 and draft PR #90 remain open.** - -**Systematic group 2a, round 14 — 2026-10-06.** [Both complete independent qualified reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-06T17-49-28-w7-systematic-group-2a-round-14-review.md) -accept the 66-path scope at `207b2716`: cold operation ownership, generated projection/pricing -precedence and actual money durability joins retain their guarantees. Contracts passes 955 tests; -Runtime passes 934, with fresh builds and strict fixtures. Both reports, the stopped static supplement, -all 3,553 actual physical entries and 153 external artifact tuples are independently audited before -Original resumes; initial launch/read/preparation gaps stay qualified. **Group 2a is accepted within -this scope. Proposed ADR-0102 approval/implementation, remaining Groups 3–6, live-fixture-gated -Steps 7–8, final Step 12, current Sonar, whole W7 and draft PR #90 remain open.** - -**Systematic group 3, round 1 — 2026-10-06.** [Both complete independent reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-06T18-43-01-w7-systematic-group-3-round-1-review.md) -verify two High acknowledgement races: late timeout refusal overrides a claimed budget decision; -invalid inline approval closes native SQLite before the aggregate pause append is acknowledged. -`a78f859a` preserves the synchronous decision claim, and `7ceae889` drains the actual durable pause -or terminal before resource release. Nineteen independent regressions are promoted; the old-code -baseline has seven expected failures among 24 cases, and the correction passes all 24. Forced -Original checks pass 23 tasks, 362 files, 8,239 tests and 12 existing skips, plus six builds and -isolation. Both complete reports, all 3,802 actual entries and 362 external artifact tuples are -independently audited with timestamp, inherited-read and exercised-guard qualifications retained. -**Fresh complete round 2 is required; Group 3, Proposed ADR-0102, remaining Groups 4–6, Steps 7–8/12, -current Sonar, W7 and draft PR #90 remain open.** - -**Systematic group 3, round 2 — 2026-10-06.** [Both complete fresh reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-06T19-25-33-w7-systematic-group-3-round-2-review.md) -verify one High: after an invalid inline approval, SIGINT during a held native pause append loses -cancellation to the paused exit. `350f7152` retains cancellation through the acknowledged terminal; -eleven permanent native controls include held cancellation writes and actual authorization claims. -The independent 25-case baseline has three expected failures; its one-guard correction and all -prepared controls pass. Original checks pass 23 forced tasks, 364 files, 8,250 tests and 12 existing -skips, with six forced builds, isolation and format. Both complete reports, all 3,635 actual entries -and 862 external artifact tuples are audited with inherited-read, environment and exercised-guard -qualifications retained. **Fresh complete round 3 is required; Group 3, Proposed ADR-0102, -remaining Groups 4–6, Steps 7–8/12, current Sonar, W7 and draft PR #90 remain open.** - -**Systematic group 3, round 3 — 2026-10-06.** [Both complete fresh reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-06T20-13-20-w7-systematic-group-3-round-3-review.md) -verify two Highs: cancellation re-enters queued cards, and cancellation during paused renderer -finalization loses its subscription, native terminal write and truthful Ink summary. `94948cd4` -latches cancellation, suppresses later cards and retains the iterator through a summary settlement -barrier and fallback resource drain. Eleven promoted regressions have seven causal old-code -failures; the corrected independent combined 61-case focus passes. Original passes 23 forced -tasks, 366 files, 8,261 tests / 12 existing skips, six builds, isolation and format. All 3,797 -actual sealed entries and 3,349 external artifact tuples are independently audited with complete -reports, changed-case reads and preparation/guard/platform qualifications retained. **Fresh -complete round 4 is required; Group 3, Proposed ADR-0102, Groups 4–6, Steps 7–8/12, current Sonar, -W7 and draft PR #90 remain open.** - -**Systematic group 3, round 4 — 2026-10-06.** [Both complete fresh reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-06T20-57-47-w7-systematic-group-3-round-4-review.md) -verify two High UI-lifetime defects: genuine budget rejection opens a surplus queued card; -cancellation or acknowledged terminal during suspension or an open card retains stale input. -`9a62d665` follows emitted rejection/terminal authority, aborts scoped Clack input and preserves -the primary terminal/durability stream. A losing idempotent rejection leaves genuine later gates -usable; ordinary rejection still continues. Twenty-five permanent controls have 16 causal -old-code failures, then all pass; the corrected combined 135-case focus passes. Original passes -23 forced tasks, 369 files, 8,286 tests / 12 existing skips, six builds, isolation and format. -Both complete reports and all 3,745 actual sealed entries / 3,260 external regular tuples are -independently audited, with helper-guard, timer-seam, SDK/platform and preparation qualifications -retained. **Fresh complete round 5 is required; Group 3, Proposed ADR-0102, Groups 4–6, -Steps 7–8/12, current Sonar, W7 and draft PR #90 remain open.** - -**Systematic group 3, round 5 — 2026-10-06.** [Both complete fresh reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-06T21-43-24-w7-systematic-group-3-round-5-review.md) -retain one verified High: authored gate timeout/autoapproval during voluntary paused finalization -can misreport the terminal, and an unpublished node/terminal append can outlive SQLite closure. -Parent independently reproduces four failures among five real native controls; a stable pause -passes. ADR-0103 is being drafted/reviewed and no dependent lifecycle implementation is approved. -`fa884a92` separately corrects the CI completion-phase oracle and the safe status JSON contract; -26 lifecycle/Clack and 22 JSON/status cases pass in independent parent preparation. Original passes -23 forced tasks, 369 files, 8,292 tests / 12 existing skips, isolation, format and diff checks. -Both complete reports/child supplements and all 3,565 sealed entries / 3,077 regular tuples are -independently audited before Original resumes. **Group 3 remains changes requested. Fresh narrow -correction reviews and the High's approved implementation are required; ADR-0102, Groups 4–6, -Steps 7–8/12, current Sonar, W7 and draft PR #90 remain open.** Independent Groups 4–6 may advance -through their own checks and review while the new lifecycle decision is pending. - -### W7 group 3 round-5 correction accepted in narrow scope — 2026-10-06 - -Two fresh independent static/runtime rounds accept the nine-path test/documentation correction -through `8e241377`. Runtime focuses pass 98 and 96 cases, with fresh ordered builds, strict inclusion -and causal cleanup/JSON controls. Parent independently audits every recorded actual artifact field -and all 1,154 source/Original receipts before releasing both freezes. The [qualified review record](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-06T22-16-35-w7-systematic-group-3-round-5-correction-review.md) -retains all preparation/read/guard/platform bounds. Full Group 3 remains changes requested: the -voluntary paused-departure High is open, ADR-0103 remains an unapproved draft and ADR-0102 remains -Proposed. Independent Groups 4–6 may proceed under the existing plan; no W7/merge acceptance is granted. - -### W7 systematic group 4, round 1 — 2026-10-07 - -Both [complete independent reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-07T06-17-17-w7-systematic-group-4-round-1-review.md) -verify one High: acknowledged activation can delete isolated corrupt identity evidence, then reissue -an old effect-turn key. The correction preserves the canonical legacy floor inside the same deletion -transaction, or refuses deletion and allocation with evidence intact. Parent focus passes 585 tests; -removing only the preservation call fails seven controls, and exact restoration passes all 51 cases. -Original passes 23 forced tasks, 371 files, 8,316 tests / 12 existing skips and six forced builds. -Both reports and all 3,067 actual sealed entries / 2,704 regular artifacts are independently audited -before Original resumes, with preparation, register and native-helper qualifications retained. -**Group 4 remains changes requested; fresh complete round 2 is required. Group 3's High, -Proposed ADR-0102, unapproved ADR-0103, Groups 5–6, Steps 7–8/12, current CI/Sonar, W7 and draft -PR #90 remain open. No additional paid generation is authorised.** - -### W7 systematic group 4, round 2 — 2026-10-07 - -Both [complete independent reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-07T07-00-05-w7-systematic-group-4-round-2-review.md) verify a High: a known SQLite suspension column -with an unknown payload discriminator could be skipped, letting reconciliation invent an interruption -terminal. The correction checks raw type equality before tolerant parsing and refuses without leases, -terminal writes or changes to either run. Thirteen new native regressions pass; removing only the guard -fails eight required-refusal controls while five genuine/matching-unknown controls pass. Original passes -23 forced tasks, 373 files, 8,329 tests / 12 existing skips, six forced builds and isolation. Parent audits -all 3,082 actual sealed entries / 2,719 regular artifacts and actual native-child proof closure before -Original resumes, preserving read/preparation/checkJs limits. **Group 4 remains changes requested; -fresh complete round 3 is required. Group 3, Proposed ADR-0102, unapproved ADR-0103, Groups 5–6, -Steps 7–8/12, final CI/coverage/Sonar, W7 and draft PR #90 remain open.** - -### W7 systematic group 4, round 3 — 2026-10-07 - -Both [independent reports](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-07T07-43-01-w7-systematic-group-4-round-3-review.md) verify a High: discovery and the state reader could trust a -streaming SQL label before validating a genuine suspension payload. Actual native reconciliation then -invented an internal failure terminal. The correction validates every stored row through the existing -parser before excluding genuine streaming events from the returned fold; persisted-history validation -includes their parsing cost. The two permanent native files pass 81 cases; with existing run-history -tests, 164 pass. Separate causal removals fail 19 discovery and 11 state-read controls; exact restoration -passes. Original passes 23 forced tasks, 373 files, 8,397 tests / 12 existing skips, six forced builds and isolation. Parent audits all 3,126 sealed physical entries / 2,763 regular artifacts, -full actual self bindings, registers and child provenance before Original resumes. The runtime report -is interrupted and its wrong store/reader API attempt is not counted as state-refusal evidence; the -complete static report and independent Parent actual-reader regressions establish that surface. -**Group 4 remains changes requested; fresh complete round 4 is required. Group 3, Proposed ADR-0102, -unapproved ADR-0103, Groups 5–6, Steps 7–8/12, final CI/coverage/Sonar, W7 and draft PR #90 remain open.** - -### W7 systematic group 4, round 4 — 2026-10-07 - -[Both fresh reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-07T09-10-29-w7-systematic-group-4-round-4-review.md) cover all 33 corrected paths / 61 hunks and ten complete test files / 5,108 lines. -No new material implementation finding remains. Three literal newline escapes in the preceding -review prose are corrected without changing its historical result. Independent runtime passes -29 files / 753 tests, five fresh builds and strict fixture/setup/config inclusion. Discovery-only -and state-only removals fail 43 and 31 controls; exact restoration passes. Parent audits both -complete reports, all 3,145 physical entries / 2,781 regular files, source receipts, self bindings, -registers and eight actual native-child proof closures before Original resumes. Read ordering, -18 historical Own-evidence revisions, capture/format repairs and increased parsing cost remain -explicit qualifications. **Group 4 is accepted within scope. Group 3's High, Proposed ADR-0102, -unapproved ADR-0103, Groups 5–6, Steps 7–8/12, final CI/coverage/Sonar, W7 and draft PR #90 remain -open. No additional paid capture or ADR-dependent implementation is authorised.** - -### W7 systematic group 5, implementation — 2026-10-07 - -The [budget replay harness](../../tools/budget-replay-compat/README.md) now runs in the mandatory GitHub CI job. Its predecessor -portable bytes are shipped independently of live installed dependencies, and all 143 source files -are checked against the actual pinned Git baseline. Native SQLite remains the documented, -invocation-hashed ABI exception. Completed evidence is bounded to three invocations; active, -interrupted, foreign and redirected evidence is preserved. Seven provenance, 17 closure and eight -retention controls pass, with seven additional finalization cases in three result groups. The -isolation probe uses explicit file-marker readiness raced against child closure instead of elapsed-time assumptions. Turbo is pinned -exactly to the cooled 2.11.4 release, with its installed local schema and unchanged task definitions. - -Forced Original `pnpm run ci` passes: 23 lint/typecheck/test tasks, 373 test files, 8,397 passing -cases / 12 existing skips, six builds, formatting, migration sync, fences and all three offline -smokes. Initial tool-lint and formatting failures are retained; neither is presented as a passing -CI run. `typecheck:tools` covers its configured TypeScript programme, not every MJS helper. -**Group 5 is implemented and awaits fresh independent reviews; it is not accepted. Group 3's -High, Proposed ADR-0102, unapproved ADR-0103, Group 6, Steps 7–8/12, final coverage/Sonar, W7 -and draft PR #90 remain open. No additional paid capture or ADR-dependent implementation is -authorised.** - -### W7 Group 5 — review corrections implemented, fresh acceptance pending — 2026-10-08 - -The [first independent systematic Group 5 review](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-08T04-58-12-w7-systematic-group-5-round-1-review.md) corroborates -secondary-evidence failure masking, asynchronous closed-stderr errors, native failed-spawn -completion before close, a same-user stale retirement pathname, and two IPC wording mistakes. -Actual callers now preserve failure precedence, join real close and write fixed diagnostics -synchronously. Retirement claims and validates physical identity plus exact records, preserving -unexpected claims for inspection. Fourteen permanent caller controls and five causal removals -pass their intended positive/negative outcomes. Forced full CI passes 23 tasks, 373 files / -8,397 passing tests / 12 skips and six builds after an initial retained DB-timeout failure and a -clean isolated 522-test DB run. This is an implementation correction; fresh Group 5 round 2, -Group 6, ADR approvals, provider captures and final whole-wave acceptance remain open. - -### W7 systematic group 5, round 2 — 2026-10-08 - -[Two fresh complete reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-08T11-44-55-w7-systematic-group-5-round-2-review.md) and independent Parent audits accept all 22 Group 5 paths / 47 textual hunks and the immutable archive at `ab8c5138`. The actual harness closes all three workers and passes twelve predecessor refusals / two positive controls; fourteen caller controls, five grouped causal removals with exact restoration, 33 independent writer cases, six preparation cases and the real 45-second watchdog pass. Both full reports, physical inventories, source/Git pins and command/read/native evidence are verified before the Root freeze is released. Retirement compares directory identity and complete parsed records by semantic equality; it does not attest exact raw record bytes or record inodes. Application-isolation, transport/read-capture, historical release-date and initial CI-failure qualifications remain in the review record. **Group 5 is accepted within scope. Group 6, Group 3's High, Proposed ADR-0102, unapproved ADR-0103, provider captures, Steps 7–8/12, final CI/coverage/Sonar, W7 and draft PR #90 remain open.** - -### W7 Proposed ADR-0103 published for maintainer review — 2026-10-08 - -[ADR-0103](../decisions/0103-a-paused-run-hands-off-its-local-producers-before-its-host-closes.md) is published with the exact eighth reviewed candidate bytes, after [two complete static reviews and qualified Parent audits](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-08T11-44-55-w7-adr-0103-eighth-proposal-review.md). It proposes engine-owned safe paused-host departure, narrow late-receipt authority, truthful money/effect disposition and CLI exit 8. **It remains Proposed; no dependent implementation is authorised before maintainer approval. ADR-0102 also remains Proposed.** Independent Group 6 work continues; the lifecycle High, provider captures, Steps 7–8/12 and final W7/PR acceptance remain open. - -### W7 systematic group 6, narrow corrections implemented — 2026-10-08 - -The remaining review-document discrepancies are corrected: 51-item diagram, five recorded security sittings, MCP consent flags, expression anchor, disclosure-reader docblock and review EOF whitespace. Behaviour-preserving leaf changes make streaming membership a Set after full persisted-row validation, flatten the replay capture-cut selector, preserve exact generated loader bytes with raw literals, spell the bounded overflow close observation inside the loop, extract the budget detail label and retain primitive-only cold-request assertion formatting. The native signal fixture also receives its explicit string parameter annotation. - -Forced full Root `pnpm run ci` passes **373 files / 8,397 tests / 12 skips**, 23 initial tasks including build dependencies, six package build targets, seven build/format tasks and all offline smokes. Explicit strict/checkJs inclusion of the changed native fixture passes separately. Sixty-three synthetic loader cases generate identical bytes and parse; 32 event/state cases keep the exact capture cuts. **These are implementation checks, not independent Group 6 acceptance. Fresh independent rounds and individual adjudication of the current 180 Sonar labels remain required. ADR-0102/0103, provider captures, Steps 7–8/12 and final W7/PR acceptance remain open. No additional paid call or ADR-dependent implementation occurred.** - -### W7 systematic Group 6, round 1 — 2026-10-08 - -Two complete independent reviews of `c6c55b3b..bbe01091` verify one new MCP-consent documentation discrepancy and two pre-existing overflow-smoke error/close-join failures. After both full Parent physical and semantic audits, the narrow correction observes native errors immediately, joins close on every failure and limits the documented no-MCP guarantee to `budget abort`. Actual offline overflow smoke, tools lint and configured tools typecheck pass. Review/capture/guard qualifications and the historical 180/current 170 Sonar boundary remain explicit in the [round-1 record](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-08T13-20-40-w7-systematic-group-6-round-1-review.md). Fresh round 2 is required; Group 6, both Proposed decisions, live captures, Steps 7–8/12 and final W7 acceptance remain open. - -### W7 systematic Group 6, recovered round 2 — 2026-10-08 - -[Two fresh independent reviews and complete qualified Parent audits](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-08T14-50-57-w7-systematic-group-6-round-2-review.md) verify missing permanent native failure regressions and the unregistered `budget abort` spelling. The actual smoke now shares its private runner with five native failure controls; separately reverting error observation or final close joining fails the mandatory smoke, and exact restoration passes. The canonical command is corrected to `budget resume --abort`. Interrupted prior evidence is not counted as a completed round. Tools lint, configured typecheck and offline smoke pass; required CI at `7c29c462` passes separately. **Fresh round 3 is required. Group 6, Proposed ADR-0102/0103 approval and dependent implementation, Group 3's High, provider evidence, Steps 7–8/12, Sonar and final W7/PR acceptance remain open.** - -### W7 systematic Group 6, round 3 — 2026-10-08 - -[Two complete cumulative reviews and a fresh current Sonar adjudication](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-08T15-45-57-w7-systematic-group-6-round-3-review.md) cover all 17 paths / 27 hunks through `82acfcac` and all 175 current keys. Parent verifies all three complete reports, physical self bindings and source/native evidence before releasing the freeze. The actual mandatory smoke passes at head/restoration and under `CI=true`; separate error-observation and close-join removals fail its permanent controls. Generated loader/body bytes are unchanged. Root full CI passes 373 files / 8,398 tests / eleven skips; exact-head GitHub required CI and coverage pass separately. No new material implementation finding remains. **Group 6 is accepted within scope. Sonar remains failed pending three caller-scoped security dispositions; the complete keyed registry and proposed texts are linked from the review. Proposed ADR-0102/0103 approval and dependent implementation, Group 3's lifecycle High, Gemini/Anthropic missing provider evidence, Steps 7–8/12 and final W7/PR acceptance remain open. No additional paid call or ADR-dependent implementation occurred.** - -### W7 ADR-0102/0103 maintainer follow-up — 2026-10-08 - -[Round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-08T18-21-45-w7-adr-0102-0103-maintainer-round-1-review.md) checks the eight maintainer items and verifies a reachable Gemini SDK prototype-key merge conflict. The corrected Proposed ADR-0102 explicitly narrows non-cap data to refuse own `__proto__` keys; it preserves the cap exception and unchanged measured subset. Proposed ADR-0103 now specifies real primary publication/delivery observation, gap/abandonment refusal and the terminal-free local-stream qualification. [Two fresh complete round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-08T18-25-00-w7-adr-0102-0103-maintainer-round-2-review.md) find no further material decision-level issue. **Both ADRs remain Proposed; the additional compatibility narrowing and lifecycle/CLI decision require maintainer approval before dependent code.** No runtime acceptance or new provider result is claimed. The supplied Gemini 3.1 Flash Lite Free Tier quota is 250K input TPM, below its 1,048,576-token window; the capture access/authorization gate remains open. Ownership/lifecycle implementation Highs, Steps 7–8/12, Sonar and W7/PR acceptance remain open. - - -### W7 latest decisions approved and completion sequence — 2026-10-08 - -The maintainer approves the latest [ADR-0102](../decisions/0102-a-measured-request-owns-its-inert-data-through-egress.md) -and [ADR-0103](../decisions/0103-a-paused-run-hands-off-its-local-producers-before-its-host-closes.md), -including their 2026-10-08 clarifications. Both are Accepted. Dependent implementation is -authorised on `development`; earlier approval restrictions remain historical records. - -Complete the remaining work in these reviewable increments: - -1. Own request data and candidate cap projections at invocation; prove actual SDK generate/stream - parity and typed refusals, then reuse the measured construction through core/media handoffs. -2. Implement engine departure, primary delivery observation, structured receipt authority and - retained exact-fence writer lifetimes; preserve bounded terminal publication. -3. Derive parked-node deadlines and integrate acknowledged CLI input/host departure, final - receipt disposition, exit 8 and the canonical diagnostic contract. -4. Obtain the two missing genuine provider records, complete Step 7 classification and Step 8 - compaction/recovery, and close Step 12 through whole-wave acceptance and document updates. - -Each increment receives an implementation commit, fresh independent review, confirmed fixes -and another fresh review before acceptance. These are implementation subdivisions of the -approved work, not replacement W7 step numbers. The maintainer authorises paid Gemini access -and necessary new provider work with a 2 USD test-call limit. Billing setup requiring a separate -legal agreement or a larger minimum prepayment remains an explicit account-setup action; no -new provider request has been made under this authorisation yet. Implementation, provider -evidence, final CI/coverage/Sonar and all six remaining W7 register items remain open until -their actual acceptance evidence lands. +## Standing rules +> **Branch from `origin/development`, never from a local `main`.** A long-lived `development` merged into +> `main` by merge commit leaves a stale local `main` behind after every PR, silently and without conflict. +> Run `git fetch` and cut from `origin/development`. -### W7 authorised provider evidence and internal ownership foundation — 2026-10-08 - -The maintainer completed Gemini billing; the selected Gemini 3.5 Flash-Lite model has paid -Tier 1 access and a 4M input-TPM quota. The existing OS-keychain account was used through stdin, -without exposing its key. One authorised Gemini request returned the genuine HTTP 400 -`INVALID_ARGUMENT` input-overflow body; its exact v1 artifact is preserved in the -[fixture index](../../packages/llm/src/conformance/fixtures/overflow/README.md). **Four of five genuine records now exist. Only Anthropic's native -`model_context_window_exceeded` response is missing.** The earlier approval sequence's -no-new-call/two-missing statements describe its publication checkpoint, not the present state. - -One free Anthropic token-count request calibrated 199,762 input tokens. One accepted Haiku 4.5 -probe ended with `end_turn`, so it is retained privately and does not establish native overflow. -Its reported 199,762 input / 1,505 output tokens imply 0.207287 USD at the published rates; this -is usage-priced evidence, not an invoice. The Gemini overflow was an unsuccessful 400 request; -no successful token usage or invoice was returned. The bounded next Anthropic probe, available -only through explicit capture purpose selection, remains subject to the maintainer's 2 USD -limit and independent capture-tool review. No automated paid retry is authorised or introduced. - -[Fresh ownership foundation round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-08T19-29-00-w7-inert-graph-round-2-review.md) -accepts only the internal inert-graph helper after descriptor/native-brand fixes and separate -mutable-alias controls. Full request/cap associations, actual SDK handoffs and core measured-round -reuse are not implemented or accepted by that record. ADR-0102/0103 implementation, the remaining -provider record, Steps 7–8/12, final CI/coverage/Sonar and all six W7 register items remain open. +- The gate is `pnpm run ci` plus `pnpm coverage:enforced`; `development` is pushed after each accepted increment + and its push CI is observed. +- `main` carries branch protection (PR required, `lint · typecheck · test` and the coverage floor required, + force-push and deletion blocked) and a `v*` tag ruleset limited to the maintain/admin roles; `release.yml` + asserts its ref is reachable from `main`. + +## History + +Finished work is recorded in its register; this page no longer restates it. + +- **Phase 2.6.5 (W0–W8)** — the [phase register](phases/phase-2.6.5-core-reliability-remediation.md), including + the W7 and W8 closing registers and merge checkpoints and the W4 start-gate ruling. +- **Phase 2.5.5** — the [phase register](phases/phase-2.5.5-hardening-and-remediation.md): the 2026-07-19 review + triage under Positioning and the traceability appendix, Waves 0–1 and the PR #81 closing list in the + [moved execution record](phases/phase-2.5.5-hardening-and-remediation.md#execution-record-moved-from-currentmd--2026-11-04), + the Wave 2 and Wave 3 Steps 1–8 merge checkpoints, and every Wave 3 increment's checkpoint and acceptance. +- **Phase 2.6** — the 2.6.F and 2.6.C implementation records and open obligations, in their + [workstream sections](phases/phase-2.6-conversational-authoring.md). +- **Phases 1, 2 and 2.5** — [phase 1](phases/phase-1-engine-and-llm.md), [phase 2](phases/phase-2-cli.md) and + [phase 2.5](phases/phase-2.5-cli-consolidation.md). +- **Repository checkpoint (recorded 2026-07-26; publication status corrected 2026-10-13):** the + [npm registry](https://registry.npmjs.org/relavium) records `relavium@0.1.1` published on 2026-06-28. The release + procedure is [ADR-0051](../decisions/0051-cli-distribution-thin-bundle-private-engine.md) / + [release-a-surface.md](../runbooks/release-a-surface.md). diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index 85822142..e2123883 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -2,7 +2,7 @@ > Status: Living -> Last updated: 2026-10-26 — Wave 2 is merged in PR #94; Wave 3 Steps 1–8 are merged in PR #95 (Step 9a.1 is independently accepted on `development` with its recorded low findings below, Step 9a.2a is independently accepted on `development` with its recorded low findings below, Step 9a.2b is independently accepted on `development` with its recorded low findings below, the native-identity increment NI independently accepted on `development` with its recorded low findings below (its Windows clause open), Step 9b.1 independently accepted on `development` with its named limits and recorded low findings below, Step 9b.2 independently accepted on `development` with its named limits, handoffs and recorded low findings below, and Steps 9c–19 not started; the PR95 increment is closed on `development` under the maintainer's 2026-10-21 round cap). The independent PR #95 review's findings are recorded below with the maintainer's 2026-10-19 owner rulings, and the masker's edge cases with the maintainer's 2026-10-21 acceptance bar and round cap and the 2026-10-22 rulings. Existing approved deferrals remain. +> Last updated: 2026-11-04 — the CLI MVP comes first (maintainer, 2026-10-26; decisions D58–D72 ruled 2026-11-04): Wave 3 is paused after Step 9b.2 and its Steps 9c–19 are pending, not declined; the order is in [current.md](current.md#next-work--the-cli-mvp). Wave 2 is merged in PR #94 and Wave 3 Steps 1–8 in PR #95; PR95, 9a.1, 9a.2a, 9a.2b, NI (its Windows clause closed on 2026-10-26), 9b.1 and 9b.2 are accepted on `development` with their recorded findings below. The independent PR #95 review's findings keep the maintainer's 2026-10-19 owner rulings, and the masker's edge cases the 2026-10-21 bar and the 2026-10-22 rulings. Existing approved deferrals remain; entries the MVP pulls forward say so. > The [merge checkpoint](phases/phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) records completed prerequisites; full M/N lifecycle and other scheduled work remain with their owners. > > Earlier triage: the Phase-2.6 rewrite triaged every open item, the 2026-07-19 full-project @@ -996,8 +996,10 @@ increment, after 9a.2b and before 9b (register row NI), rather than a deferral; NI is independently accepted on `development` at `fc5af701` (2026-10-25, [acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)); the item stays open until the Windows clause below closes it after three consecutive green advisory Windows runs on pushed heads. +**Closed 2026-10-26:** the third consecutive green advisory Windows run (push CI run 40276471445 at `5a328bca`, probe +`0 of 200`) closed the Windows clause ([NI acceptance](phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-acceptance--2026-10-25)); the scoped limits below stay open. -- [ ] **Identity checks compare `ino` as a JavaScript number, which Windows file IDs overflow.** *High; security, +- [x] **Identity checks compare `ino` as a JavaScript number, which Windows file IDs overflow.** *High; security, Windows.* `statSync`, `lstatSync`, `fstatSync` and `FileHandle.stat` without `{ bigint: true }` return `ino` as a number, but on Windows `ino` is the volume's 64-bit file ID; above 2^53 neighbouring IDs round to the same value, so a replaced file or directory can compare as the original. `apps/cli/src/engine/tool-host/fs-policy.test.ts`, @@ -1062,6 +1064,8 @@ surface ships. CR-95's long continuation and the exact-capacity residual remain ADR-0081's role boundary; ids unique per session; validity proven by the live conformance suite; a user-scoped `[preferences]` toggle. *(high · packages/core/src/engine/agent-session.ts, turn-messages.ts; `CR-70`, ADR-0095)* + **Scheduled 2026-11-04 (D67):** v0.2.1, right after the CLI MVP's v0.2.0, designed together with ADR-0096 §5's + in-turn overflow continuation ([current.md](current.md#next-work--the-cli-mvp)); the trigger above is unchanged. - [ ] **Binding a budget allowance to its model, and giving it an expiry (`CR-94`, remainder).** *Medium.* [ADR-0097](../decisions/0097-a-budget-approval-is-an-allowance-not-an-exemption.md) builds the minimum @@ -1090,6 +1094,8 @@ surface ships. CR-95's long continuation and the exact-capacity residual remain - **Trigger.** The first surface that needs to show, or resume from, what a failed turn did — a coding assistant replaying a half-finished edit, or an audit view over a session's tool use. *(medium · apps/cli/src/chat/persister.ts, packages/core/src/engine/agent-session.ts; `CR-71`, ADR-0095)* + **Pulled into the CLI MVP 2026-11-04 (D67, MVP-4):** a failed, denied or aborted turn stays in context as a + content-free interruption marker (ADR-0095 §1), behind an ADR-0095 amendment drafted in Gate 0. - [ ] **Two at-rest residuals `secure_delete` does not cover: a blocked WAL checkpoint, and pages freed before the upgrade.** *Low.* Recorded 2026-09-18 with the `secure_delete` decision, and sharpened the same day after a @@ -1926,6 +1932,9 @@ so directly: these "should remain visible rather than disappear behind the green > **Found while fixing 2.6.C's F1** (the `/models` reseat blanking the alt-screen viewport). Deliberately scoped > OUT of that fix, and recorded here so it is tracked work rather than a discovery that gets lost. +> +> **Scheduled 2026-11-04:** the CLI MVP's MVP-2 repaints a resumed session's transcript +> ([current.md](current.md#next-work--the-cli-mvp)); close this entry when that lands. `relavium chat-resume ` restores the model's context from `history.db`, but opens on an **empty viewport** — the prior conversation is never repainted. This is **not** the 2.6.F regression F1 is: it has been @@ -2862,3 +2871,5 @@ future test cannot silently re-acquire it. [approved execution plan](phases/phase-2.5.5-hardening-and-remediation.md#proposed-implementation-increments) and [Wave 5a owner](current.md#wave-5a--paint-before-you-build) retain the split; close only after actual word-wrap implementation and regression evidence land. + **Moved 2026-11-04 (D65):** into the CLI MVP's MVP-2 ([current.md](current.md#next-work--the-cli-mvp)); the Wave 5a + ownership ends, and the close condition above stands. diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index 7dd35d71..fbfe63d7 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -1,6 +1,6 @@ # Phase 2.5.5 — Hardening and Remediation -> Status: 🚧 **Open — Wave 2 merged as PR #94 (2026-10-15); Wave 3 Steps 1–8 merged as PR #95 (2026-10-18); Wave 3 Steps 9–19 and later waves remain open.** Live progress and execution order are canonical in +> Status: 🚧 **Open — Wave 2 merged as PR #94 (2026-10-15); Wave 3 Steps 1–8 merged as PR #95 (2026-10-18); Wave 3 is paused after Step 9b.2 since 2026-10-26 behind the CLI MVP ([priority reset](#mvp-priority-reset--2026-11-04)); Steps 9c–19 and later waves remain open.** Live progress and execution order are canonical in > [../current.md](../current.md#execution-order--phase-255--phase-26-temporary); this banner does not > restate them. A full-project multi-agent code review completed > 2026-07-19 at HEAD `f88b0e8` — 377 findings, 373 surviving two-lens adversarial @@ -3677,6 +3677,9 @@ No credential values or request captures belong in tracked evidence. > maintainer's approval; the 2026-10-19 amendment is recorded in > [Wave 3 Steps 9–19 preflight review and maintainer rulings](#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19). > Steps 1–8 merged as PR #95; live progress is canonical in [current.md](../current.md#wave-3--clear-the-ground). +> *Paused 2026-10-26 after Step 9b.2:* Steps 9c–19 are pending behind the CLI MVP; the recorded splits of rows 10b, 11b, +> 13a–13c, 15a, 15c, 16b and 17a that move into the MVP, with their order amendments, are in the +> [MVP priority reset](#mvp-priority-reset--2026-11-04). | Step | Scope, finding IDs and principal verification | |---|---| @@ -9398,11 +9401,11 @@ plan amendment (row PR95) and the recorded splits; ID coverage is unchanged apar | Area | Ruling | | --- | --- | -| Acceptance | Each increment keeps the existing rule: implement, author gates, commit, then NEW independent complete cumulative review rounds until two consecutive rounds are clean after the last material fix. Comment- or documentation-only nits do not reset the count. Security-touching increments add a separate scoped sitting: PR95, 9a.1, 9a.2, 9b, 9c, 9d (whole Step 9 custody), 10a, 10c, 11b, 13c, 14b, 15c, 16a, 16b, 16c (whole Step 16), 18 and 19b (whole wave). PR95 instead follows the maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)). **Amended 2026-10-23 (maintainer, after Step 9a.1's twelfth round):** from Step 9a.1's round 13 on, every increment is accepted after one NEW independent complete cumulative round that reports no blocker, high or medium finding and no regression; that round's low findings are recorded in [deferred-tasks.md](../deferred-tasks.md) with their file, a minimal example and a re-check note rather than fixed in the loop, and nits are optional. A high or medium finding is still corrected and followed by a new round, and the separate security sittings above are unchanged. | -| Publication | `development` is pushed after each accepted increment, so its actual CI (including the four same-tarball platform consumers) is observed per increment. The Wave 3 PR follows Step 19. | +| Acceptance | Each increment keeps the existing rule: implement, author gates, commit, then NEW independent complete cumulative review rounds until two consecutive rounds are clean after the last material fix. Comment- or documentation-only nits do not reset the count. Security-touching increments add a separate scoped sitting: PR95, 9a.1, 9a.2, 9b, 9c, 9d (whole Step 9 custody), 10a, 10c, 11b, 13c, 14b, 15c, 16a, 16b, 16c (whole Step 16), 18 and 19b (whole wave). PR95 instead follows the maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)). **Amended 2026-10-23 (maintainer, after Step 9a.1's twelfth round):** from Step 9a.1's round 13 on, every increment is accepted after one NEW independent complete cumulative round that reports no blocker, high or medium finding and no regression; that round's low findings are recorded in [deferred-tasks.md](../deferred-tasks.md) with their file, a minimal example and a re-check note rather than fixed in the loop, and nits are optional. A high or medium finding is still corrected and followed by a new round, and the separate security sittings above are unchanged. **Amended 2026-11-04 (D64):** tiered for the CLI MVP — see the [MVP priority reset](#mvp-priority-reset--2026-11-04). | +| Publication | `development` is pushed after each accepted increment, so its actual CI (including the four same-tarball platform consumers) is observed per increment. The Wave 3 PR follows Step 19. **Amended 2026-11-04 (D61):** a scoped `development`→`main` checkpoint follows the [MVP priority reset](#mvp-priority-reset--2026-11-04); the Wave 3 PR still follows Step 19. | | Plan shape | PR95 + 9a.1, 9a.2, 9b, 9c, 9d + 10a–c + 11a–b + 12a–b + 13a–c + 14a–b + 15a–c + 16a–c + 17a–b + 18 + 19a–b (29 increments). Each split is recorded before its step starts. **Amended 2026-10-24 (maintainer):** 9a.2 is split into 9a.2a and 9a.2b, so the plan has 30 increments ([9a.2 preflight decisions](#wave-3-step-9a2-preflight-decisions--2026-10-24)). **Amended 2026-10-24 (maintainer, E6):** the native-identity increment NI is added after 9a.2b, so the plan has 31 increments ([9a.2a acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)). **Amended 2026-10-25 (maintainer, L8):** 9b is split into 9b.1 and 9b.2, so the plan has 32 increments ([9b preflight decisions](#wave-3-step-9b-preflight-decisions--2026-10-25)). | | Reassignments | G54 moves from row 17 to 11a; #311's implementation moves to 10b and its certification stays in 17a; G36/G46 are re-scoped in 10a to the branch that replaces the raw-stack branch and are certified in 17b; promotion of the Steps 6–10 commands.md sections moves from Step 19 to 10c. | -| Interim release guard | 9a.1 adds a fail-closed check to the `release.yml` publish job that refuses any non-release candidate. Consequence, acknowledged: no npm publication, including a v0.1.x patch, until 9d and Wave 4b production custody exist. | +| Interim release guard | 9a.1 adds a fail-closed check to the `release.yml` publish job that refuses any non-release candidate. Consequence, acknowledged: no npm publication, including a v0.1.x patch, until 9d and Wave 4b production custody exist. **Amended 2026-11-04 (D59):** to be superseded by a pre-1.0 release ADR (option B, sunset at 1.0 or when 9c/9d land); the guard stays in force until that ADR is Accepted and its guard change passes a security sitting ([MVP priority reset](#mvp-priority-reset--2026-11-04)). | | PR95 scrubber residuals | Fixed before 9a: the `api-key:Bearer`/`Basic` leak; raw `?`/`#` userinfo masked unless the text before it is host-shaped and an `=` assignment separates it from the authority `@`, in both the LLM leaf and core's twin predicate (core's other documented differences are recorded, not removed); an extended finite scheme list that masks colon-bearing userinfo for the new schemes; the scanner invariants, prefilter and witnesses; the load-sensitive 8 MiB timing case. LOW-1 and LOW-2 become approved deferrals with their recorded triggers. | | Step 9 mechanics | The encrypted plaintext container is an in-house strict USTAR (regular files, repository-relative paths) recoverable with standard `gpg -d \| tar -x`; GnuPG ≥ 2.2.8 becomes a prerequisite of local `pnpm run ci` and of the CI Ubuntu runner, documented in local-dev-setup.md, and its controls fail rather than skip when it is absent; release-mode staging artifacts are retained 30 days, package-test artifacts 7; product-copy map negative controls run as `check:map-controls` locally and in a required uncached CI job; the private manifest keys final JS as `dist/.js`; the production recipient must be an OpenPGP v4 key with a cv25519 or RSA≥3072 encryption subkey, a Wave 4b prerequisite recorded in [deferred-tasks.md](../deferred-tasks.md#wave-3-approved-later-wave-ownership--2026-10-16). | | Step 10 compatibility | Verbose frames use a same-build allowlist (a tsup `renderChunk` substitution of a fixed-length placeholder, so no second build or post-finalization write) and appear only on classified non-infrastructure diagnostics; `--quiet` treats live progress as decoration on TTY surfaces; excess positional arguments are refused on every non-variadic command; plain `note:` stderr lines become `{type:'diagnostic', code, message}` records in JSON mode; `help [path] --json` returns the discovery DTO; human `--help` keeps Commander behaviour for unknown grammar; deprecated aliases emit one structured JSON notice or one fixed human line kept under quiet and are hidden from human help listings. | @@ -10062,6 +10065,13 @@ diagnosed mechanism, while every pair stayed distinct as exact identities, and t jobs, the advisory Windows job among them. Its probe logged `0 of 200 replacement pairs collide as Numbers; 0 read negative`, and the repeated replacement tests passed. +**Windows clause, run 3 of 3 (2026-10-26) — closed.** Push CI +[run 40276471445](https://github.com/HodeTech/Relavium/actions/runs/40276471445) at `5a328bca` passed all eighteen +jobs, the advisory Windows job among them. Its probe logged `0 of 200 replacement pairs collide as Numbers; 0 read +negative`, and the repeated replacement tests passed. With three consecutive green advisory Windows runs on pushed +heads, the Windows clause and the NI entry in [deferred-tasks.md](../deferred-tasks.md#native-file-identity-on-windows--approved-increment-2026-10-24) +are closed; NI's scoped limits stay recorded there. + ### Wave 3 Step 9b preflight decisions — 2026-10-25 A read-only preflight (four readers and a synthesis at `c8c0d41b`) mapped Step 9b's obligations, the code it meets and @@ -10614,4 +10624,499 @@ The round's verified findings were twelve lows and one nit: [deferred-tasks.md](../deferred-tasks.md#wave-3-step-9b2--recorded-low-findings-2026-10-26), each with its example and re-check. -Step 9c is next. +Step 9c was next; Wave 3 was paused on 2026-10-26 behind the CLI MVP ([MVP priority reset](#mvp-priority-reset--2026-11-04)). + +## MVP priority reset — 2026-11-04 + +**Wave 3 is paused after Step 9b.2.** On 2026-10-26 the maintainer stopped the Step 9c preflight and redirected the +project to an MVP: features and bugfixes that a CLI user feels, in `apps/cli` and `packages/core` first. Steps +9c–19 (25 of the 32 increments; the plan grew from 29 with the 9a.2, NI and 9b splits) are **pending, not +declined**: each keeps its row, its recorded ADR amendments and its security sittings. The live order is +canonical in [current.md](../current.md#next-work--the-cli-mvp); this section records the evidence, the decisions +and the amendments to this register. + +**Evidence.** A read-only multi-agent review at `development` `5a328bca` read every roadmap document and the code: +eight scoped readers (661 open items), eight adversarial verifiers (three re-run on 2026-11-04 after a spend +limit), a synthesis and a completeness critic. The detailed reports and a Turkish decision register stay local and +Git-ignored. Its main findings, each re-checked against the code before the decisions: + +- The engine is solid; the agent and presentation layers are not. The built-in chat agent grants only + `read_file`, `list_directory` and `git_status` (`apps/cli/src/chat/default-agent.ts:35`) and starts in `ask` + (`apps/cli/src/chat/chat-mode.ts:31`); no edit, search or find tool exists; `run_command` is refused before any + approval unless allowlisted (`packages/core/src/tools/registry.ts:762-770`). +- Run results are invisible on the human surfaces (no outputs, no run id, no finished-run listing, crashed runs stay + `running`); replies are raw markdown; a resumed chat opens empty; Anthropic chat turns default to a 4,096-token + output estimate (`packages/llm/src/output-cap.ts:37`). +- Nothing newer than `0.1.1` can be published: the Step 9a.1 guard sets `process.exitCode = 1` before evaluating + (`tools/cli-package/release-guard.mjs:13`), and it exists only on `development` — `main` has no guard. +- The published `0.1.1` prints unsanitized model output in its `relavium run` view (#56); `LICENSE` §2 grants no + usage right without a written agreement. + +**Decisions (maintainer, 2026-11-04).** D58–D72 are recorded in the +[decision table](../current.md#wave-gating-decisions) with their rulings. In short: the larger MVP (v0.2.0 after +MVP-1…MVP-10, including conversational authoring, web, the Home hub and MCP management; milestone M3.5); release +path B (D59); the license unchanged (D60); a scoped `development`→`main` checkpoint now (D61); the default agent +may write, with `[chat].default_mode` (D62); a `run_command` approval card with remembered grants outside the +repository (D63); a tiered acceptance bar (D64); six recorded splits plus #61 (D65); `current.md` reduced to live +status (D66); CR-71 un-deferred and CR-70 after the release with ADR-0096 §5 (D67); first-run defaults and one +keymap ADR (D68); authoring → web → Home → MCP (D69); no pre-release (D70); the parity items in the MVP (D71); no +action on `0.1.1`'s #56 (D72). + +**Recorded splits (D65), ID coverage unchanged.** Each split keeps its row's recorded ADR amendment and security +sitting, and lands in the named MVP milestone: + +| From | Split | Lands in | Order amendment | +|---|---|---|---| +| 10b | Excess positional arguments refused on every non-variadic command | MVP-2 | — | +| 15a | #65: tabs expanded to 8 cells before width measurement | MVP-2 | Lands before #58/#62/G40/G51 (row 15's strict arrow amended) | +| 15c | #63: 80×24 on both chat surfaces with no hidden affirmative input there or on the Home strip (approved ADR-0057 amendment, approval-path sitting); with it, the Home strip row budget (2.6.F's open obligation) | MVP-3 | Lands before 15a/15b's remaining items | +| 11b | Ctrl-C at a pending approval rejects it | MVP-4 | Lands before 11a. The 2026-10-19 "maps to `/cancel`" wording is superseded by the keymap ADR (D68): reject first, then the ordinary interrupt semantics; the approval-path sitting stays | +| 16b | #51: the persistent, current-state keyless nudge (keychain/config sitting) | MVP-2 | Lands before 16a | +| 17a | #24: the one-shot stdin hint | MVP-2 | — | +| 13a–13c | G30: chat HTTP authority under ADR-0112 (egress/config-trust sitting) | MVP-7 | Row 13's "after decomposition" no longer holds: 13a/13b land before Step 12; 13c is re-scoped when MVP-7 starts because 12b's shared helpers do not exist yet | +| Wave 5a | #61: full-screen word-wrap | MVP-2 | Its Wave 5a ownership ([later-wave ownership](#existing-later-wave-ownership-and-approval-gate)) ends | + +**Amendments to the 2026-10-19 rulings** (dated, the original text kept): + +- *Acceptance (D64):* tiered. An increment touching an approval or consent path, egress, keys or secrets, + filesystem writes, a new terminal sink for untrusted content, the lease or resume paths, or release and + publication keeps the 2026-10-23 bar — a rigor pass, one NEW complete cumulative round with no blocker, high or + medium finding and no regression, and a separate security sitting. Other UX, rendering and command increments are + reviewed per MVP milestone: author self-review, `pnpm run ci` plus `pnpm coverage:enforced`, and one independent + scoped round with no high or medium finding; lows go to [deferred-tasks.md](../deferred-tasks.md). +- *Publication (D61):* a scoped `development`→`main` checkpoint follows this reset; the Wave 3 PR still follows + Step 19 when the wave resumes. +- *Interim release guard (D59):* to be superseded by a new pre-1.0 release ADR (option B) with a sunset at 1.0 or + when 9c/9d land, whichever comes first. The guard stays in force, in code and procedurally, until that ADR is + Accepted and its guard change passes a security sitting. + +**Recorded risks.** The published package grants no usage right (D60). The published `0.1.1` keeps #56's +unsanitized `relavium run` output; the maintainer accepted it because the product has no users yet (D72). + +**NI's Windows clause is closed** (2026-10-26): run 3 of 3 is recorded in the +[NI acceptance](#wave-3-ni-acceptance--2026-10-25). + +## Execution record moved from current.md — 2026-11-04 + +> Moved verbatim from `docs/roadmap/current.md` on 2026-11-04 (decision D66) when that page was reduced to live +> status and order; only relative links were rewritten for this directory and heading levels lowered by one. +> It keeps the 2026-07-26 execution plan's baseline, axes, Waves 0–1 and the PR #81 closing list — the only +> definition of `#W15-1`…`#W15-24`, which code comments cite. The live order is in [current.md](../current.md). + +### Execution order — Phase 2.5.5 + Phase 2.6 (temporary) + +> **Temporary section — delete it when both phases close.** It owns **ordering only**. Every item's +> definition, size, file list and finding ids live in its phase file — +> [phase-2.5.5](phase-2.5.5-hardening-and-remediation.md) (176 work items, sub-streams A–I) and +> [phase-2.6](phase-2.6-conversational-authoring.md) (15 open workstreams) — and nothing below +> restates them. Items are cited as ` · ()`, which +> greps directly against the phase file. +> +> Derived 2026-07-26 from a full pass over both phase files, their own sequencing sections, and the live +> tree. Where it departs from a phase file's own prose, see [Departures](../current.md#departures-from-the-phase-files-own-sequencing). + +#### Ordering root — one baseline ✅ **discharged 2026-07-26** + +The plan opened on a divergence neither phase file modelled: Phase 2.5.5's whole backlog is written against +`f88b0e8`, which existed only on `development` behind PR #76, so every `file:line` citation in it resolved +against a tree nobody was branching from. **PR #76 and #77 are both merged** — `f88b0e8` is on `origin/main`, +`origin/development` and `origin/main` held identical trees with no open PRs as of 2026-07-26. Every +citation in this plan now resolves. (PR #79 and PR #80 have opened since.) + +One consequence survives and is worth keeping until the phases close: + +> **Branch from `origin/development`, never from a local `main`.** A long-lived `development` merged into +> `main` by merge commit leaves a stale local `main` behind after every PR, silently and without conflict. +> Run `git fetch` and cut from `origin/development`. + +Also settled while discharging this: `main` now carries branch protection (PR required, `lint · typecheck · +test` required, force-push and deletion blocked) and a `v*` tag ruleset restricting tag creation to the +maintain/admin roles — closing the Phase-0 obligation that made `ci.yml`'s own "REQUIRED status check" +comment untrue. `release.yml` asserts its ref is reachable from `main` before `pack`, so **2.5.5.H's `G26` +is closed ahead of Wave 0**. + +#### The three axes this order is built on + +1. **Irreversibility** — a defect whose damage a later fix cannot undo jumps every queue. That is Wave 1: + a credential already written into a durable `--json`-visible row, a billed turn lost after the reply + rendered, model-controlled ANSI inside the consent card, and the states where the cost cap is silently off. +2. **File ownership** — whoever edits a file next inherits everyone else's rebase. Every god-file + decomposition and naming/convention ruling lands *before* the 2.6 workstream that piles onto it, and + eleven items are pulled across the phase line because they edit the exact function another item restructures. +3. **Certification** — a wave that discharges a phase obligation certifies it while the evidence is fresh, + which is why 2.5.5's exit criteria 1–3 close in Wave 2 rather than at the end. + +Waves are readable single-branch sequences. Concurrency is expressed as *"these queues are file-disjoint, +any order"* — never as headcount. + +```mermaid +flowchart TD + W0["Wave 0 — One true baseline
baseline ✅ · CI truth · numbers"] + W1["Wave 1 — Stop the bleeding ✅
3 CRITICALs · cost cap · ADR-0074"] + LEDGER["#W15-1 — realized-cost ledger ✅
ADR-0076 + ADR-0077"] + P265["Phase 2.6.5 — Core reliability
51 CR items · 8 P0 blockers behind ADR-0078–0084
W0–W8 complete · W3 blocker closed in W8
W4 merged — the hostile-MCP class, ADR-0088
W5 merged — media correctness, ADR-0089+0090
W6 merged — authoring correctness, ADR-0091–0094
W7 merged — PR #90 · W8 merged — PR #92
49/51 closed · approved deferrals retained"] + W2["Wave 2 — Shut the doors ✅
PR #94 merged · fs jail · secrets · config trust
2.5.5 EXIT 1–3 satisfied"] + W3["Wave 3 — Clear the ground
god-file decomposition · CLI net"] + W4a["Wave 4a — The spine
2.6.A/D/H/K + 2 ADRs"] + W4b["Wave 4b — Money floor & close-out
2.5.5.B/C/F/H drain · v0.2.0"] + W5a["Wave 5a — Paint before you build
render-v2 · theme · config"] + W5b["Wave 5b — Home becomes the product
2.6.G/I/J · then i18n sweep"] + W6["Wave 6 — Hands, voice, lineage
2.6.M/B/E + 2.6.N foundation"] + W7["Wave 7 — Orchestration & the gate
2.6.O/P · go/no-go → Phase 3"] + W0 --> W1 --> LEDGER --> P265 --> W2 --> W3 --> W4a --> W5a --> W5b --> W6 --> W7 + W3 --> W4b + W4b -.->|v0.2.0 release| W7 + W4a -.->|file-disjoint lanes| W4b +``` + +#### Wave 0 — One true baseline + +Collapse the divergence, make the required gate execute the artifact it ships, and fix the review +checklists before ~30 security-gated PRs are reviewed against them. + +1. ✅ **Merge PR #76** (2.6.Q P1–P5, ADR-0071/0072, the three picker bugs, the param self-heal) — **done + 2026-07-26**, along with #77 (this plan + the Phase-2.5.5 opening + `release.yml`'s `G26` ancestry gate). +2. ✅ **Reserve numbers** — **done**, recorded here: migrations end at `0012` and ADRs at `0072` (verified), + so `0013` = **ADR-0074 §4's conservative money columns** (`agent_sessions.total_conservative_microcents` + `session_costs.conservative_microcents`) — the number was freed when D8 ruled against the approval-preview scrub and is RE-USED here rather than left as a hole · `0014` = 2.5.5.C's enum CHECKs (#101) · `0015` = 2.6.H · + `0016` = 2.6.G's pins · `0017` = 2.6.N lineage; **ADR-0075+** for the nine unwritten 2.6 ADRs (`0073` = the history.db migration lock, `0074` = durable conservative budget commitments — both landed in Wave 1). +3. ✅ **One `ci.yml`/`turbo.json`/`tsup.config.ts` PR** — **done 2026-07-29 (PR #80)**, all eight items: + 2.5.5.H · the required gate never runs the compiled binary + undeclared `drizzle` output (#294, #315) → + local `pnpm run ci` vs `ci.yml` divergence (#312 — and `pnpm ci` is shadowed by a pnpm builtin, so the + script was unreachable by the name everyone types) → the coverage-floor **ruling and its implementation** + (#296, #152) → the `(advisory)` labels (#320) → `THIRD_PARTY_EXTERNAL` (G27, #248) → bundle-closure + single-chunk assert (#314) → `sync:models:check` (#317). *(`release.yml`'s ancestry check, `G26`, landed in + #77; its tag-protection half is configured.)* The PR also closed the four blockers from the 2026-07-29 + roadmap review, and its own CI caught two real defects in it — a TS4111 that `turbo run typecheck` does not + cover, and `pnpm ci` being unreachable because pnpm reserves `ci` as a builtin. +4. ✅ **2.5.5.F · three skills' hardcoded foreign-project paths** (#162) — **done 2026-07-29.** Filed as docs, + actually a functional bug: they pointed at `/Users/dev/…/Agent-Organizer/`, so `add-package`'s `mkdir -p` + would have created a tree outside the repo. All three now anchor on `$(git rev-parse --show-toplevel)`. +5. ✅ **One markdown PR** — **done 2026-07-29**: `packages/mcp` added to all five inventory sites (#128, #129, + #153, #163, #254), the reviewer agent's secrets item extended to the shipped CLI floor (#164), the + security-review skill's SSRF step now requires reusing the shared guard rather than re-deriving one (#167), + and the i18n clause settled by the D5 ruling (#74, #260, #134). +6. ✅ **The locale-bar amendment** (D5, `en`+`tr`) and the **snapshot regen** (D3) — **done**, propagated to + all seven locale sites and merged as PR #79 (two price rises taken, nine retirements accepted). + +> ✅ **Wave 0 is COMPLETE (2026-07-29).** The required gate now executes the binary it ships, +> `apps/cli/drizzle/**` is a declared turbo output, and `lint · typecheck · test` + +> `engine coverage floor (llm, mcp)` are both required checks under branch protection. In phase terms this +> closes **8 of 2.5.5.H's 14** tasks and **3 of 2.5.5.F's 20**. **Wave 1 followed and is also complete +> (2026-07-30, PR #81) — see below; Wave 2 is next.** + +#### Wave 1 — Stop the bleeding + +All three CRITICALs, plus the states where `max_cost_microcents` is silently not a cap. Four file-disjoint +lanes, any order between them, plus the money hole they left open. + +**✅ Wave 1 is COMPLETE — lanes (a)–(d) on 2026-07-30 (PR #81), lane (e) on 2026-08-09 (`#W15-3`).** + +> **Superseded 2026-08-09 — kept because the correction is the record.** Lane (e) closed with `#W15-3`: +> `/cost --release` makes §1's escape hatch reachable, and `/cost` renders the durability state. The +> retraction below was accurate when written and is no longer; it stays so the history of the false +> completion claim is not quietly erased. +> +> **Retraction (2026-07-30).** This section previously said Wave 1 was complete and ADR-0074 fully implemented. +> That was wrong on lane (e) and I am correcting it rather than leaving it: **§1's release escape hatch is +> exposed on `GovernorWiring` but reachable from no command, REPL action or Home affordance**, and `/cost` never +> renders `conservativeDurabilityBroken`. §1 names this exact state as forbidden — "a single usage-less response +> would permanently shrink a long-lived chat's cap with no way out" — and this PR REMOVED the accidental escape +> a restart used to provide. A section whose compensating bound is unreachable is not implemented, and marking +> it done made four other documents inherit the error. Two independent reviews flagged it; both were right. +> +> Lane (e)'s remaining work, and two defects it exposed that ADR-0074 does not cover, are tracked in +> **[PR #81's closing list](#pr-81s-closing-list-and-the-two-items-that-need-an-adr)** below. Every lane below is verified against the shipped code, not +against its own task text — the check for each is stated inline. Two gaps are recorded rather than closed; both +are named in their lane and marked in the source, and neither is a shipped defect: + +1. the engine-level composition test for ADR-0074 §3's abort-breakable media-job hold (`engine.ts`, at the abort + listener) — the governor's own state machine is mutation-verified, the composition with `#countRunning`'s + termination gate is not; +2. the CLI-layer test that `restoreConservativeCost` is wired through `buildSessionRuntime` → `GovernorWiring` + (`session-host.test.ts`) — `AgentSession.resume` calling the hook IS mutation-verified in `packages/core`; + only the spread that supplies it is unproven. + +- ✅ **(a) `registry.ts`** — 2.5.5.A · **redact tool-approval previews like `sanitizeInput()` already does** + (#91). *(The originally-planned migration 0013 scrubbing already-persisted previews was ruled OUT — see D8.)* *Closed: `previewFor()` now scrubs the DISPLAY copy while `rawPreviewFor()` keeps an + `unredactedPreview` for in-process classification — splitting the two uses, because a whole-string scrub + flipped the protected-path decision and a per-segment one reopened the leak. Verified: `registry.ts` and + `tools/types.ts` both carry `unredactedPreview`.* +- ✅ **(b) `packages/db` chain, strict** — 2.5.5.C · `SQLITE_BUSY_SNAPSHOT` (#100) → async backoff replacing + the blocking `Atomics.wait` (#226) → **the chat-persister safety net** (#228, CRITICAL) → + serialize `runMigrations` (#99) co-landed with the unconditional 0600 self-heal (#28, #33) → + `createClient` typed error + PRAGMA docstring (#104, #105). `deprecationDate` crash guard (G1) is a + standalone one-liner, droppable anywhere. +- ✅ **(c) render/signals** — 2.5.5.I · **terminal-control sanitization to the four boundaries it never + reached** (G34, G44, #56, #57, CRITICAL — one PR at the shared helper, its own security review) → + `SIGTSTP`/`SIGCONT` + onboarding signal-registration ordering (G0, #50). *Closed at all four boundaries the register names, verified by call count: + `RunApp.tsx` (#56) 9, `final-summary.ts` (#57) 6, `render-error.ts` (G34) 4, `clack-prompter.ts` (G44) 4. G0's + run/gate `SIGTSTP`/`SIGCONT` is `render/run-job-control.ts`; #50 is closed by ordering — + `drive-home.tsx` subscribes signals at `:808`, before `runOnboardingWizard` at `:855`.* +- ✅ **(d) money, split across two PRs** — `budget-governor.ts` only: 2.5.5.A · reservation ledger under + `max_parallel` (G38) → re-armed `budget:warning` + projected `thresholdPct` (G39, G47, G49) → **2.6.Q's + strict-cap half**. Then `fallback-chain.ts`/`cost-tracker.ts` only: 2.5.5.B · `maxRetries: 0` + jitter + + `Retry-After` (#276, #279) → narrowed `#emitSuccess` catch + `CostTracker` bounds (#194, #198) + + **2.6.Q's realized-cost half**. + +- 🔶 **(e) ADR-0074 — durable conservative budget commitments (§2–§5 shipped; §1 INCOMPLETE)** · the money hole the other four lanes leave open: + a bounded estimate retained when a provider may already have billed but returned no trustworthy usage was + kept only in memory, so a crash or a resume reopened a strict cap against money that may already be owed. + §5 (a forward-compatible stored-event READ, the precondition for emitting anything new) → §2 (the + `budget:estimate_committed` contract, the governor's durability barrier, the checkpoint fold and the resume + restore) → §3 (freeze an async media job's accepted units + price) → §4 (the session's conservative + aggregate + per-model attribution + migration + transactional persister write). + +> 2.6.Q's cost-cap item is **split**: its task text spans both `budget-governor.ts` and `fallback-chain.ts`'s +> `#emitSuccess` — the same catch 2.5.5.B narrows. The two halves join two different PRs, not one. +> **Closes M2.5.5-1.** +> +> **Outstanding inside lane (e) — restated at close, because §4 changed the facts.** ADR-0074 §1 ties a +> commitment's release to the surface that renders it. +> +> **Rendering: done on all three surfaces.** The run TUI and `relavium logs` shipped with §2; `/cost` shipped +> with §4's fold and reads *estimated, possibly billed*, with `(no completed call)` for a model that has one +> without a completed call. All three are mutation-verified. +> +> **Release: reachable, as of `#W15-3` (2026-08-09).** `/cost --release` on the chat surface — the one that +> renders the amount, which is what §1 ties it to — clears the durable per-model and aggregate columns first +> and the live governor second, so a released commitment does not return on the next `chat-resume`. `/cost` +> also renders the durability state. This note previously said no affordance called it; that was true and no +> longer is. +> +> The reserved `budget:estimate_released` event stays **reserved and unemitted**, and that is a decision, not +> an omission: the session path restores its conservative total from COLUMNS rather than by replaying events, +> so zeroing them is what makes the release durable there. The event is needed only by a workflow-run release +> surface, which does not exist — a run is not long-lived, and `on_exceed: pause_for_approval`'s per-node +> bypass or raising the YAML cap are §1's own analogy. §1's dated note in +> [ADR-0074](../../decisions/0074-durable-conservative-budget-commitments.md) carries the same statement. + +#### PR #81's closing list, and the two items that need an ADR + +Two independent adversarial reviews of PR #81 (2026-07-30) found that the cap's core safety claim does not yet +hold across a process boundary. + +**Almost all of this belongs to PR #81 itself, not to a later wave** — a first framing filed it as one, which +was wrong. The test is simple: this PR either ADDED the code a finding is in (`background-failure.ts`), or +claimed to have closed the area it sits in (the terminal-safety boundaries, ADR-0074 §1), or REMOVED a +protection without replacing it (§1's escape hatch, which a restart used to provide). Shipping a regression and +deferring its fix is not a schedule decision. + +**Only §A is genuinely separable.** Both of its items need an append-only ADR before any code — one changes what +"spent" means across a crash, the other supersedes a decision ADR-0074 §5 already recorded. Those earn their own +PRs. Everything from §B down is #81's to close. + +Ordered by whether the repo currently states something untrue, then by blast radius. + +**Every item names the check that would close it, not just the defect.** + +> **Status — 24 of 24 closed and MERGED, across two PRs.** Everything marked ✅ below is fixed and +> break-verified with the mutation confirmed applied; the two PRs are recorded separately because the closure +> dates differ: +> +> - **23 items merged to `main` via PR #81** (2026-08-09). +> - **`#W15-1`, the last one, closed 2026-08-10** behind ADR-0076 + ADR-0077 (see §A) and **merged to `main` +> via PR #82** (2026-08-11), together with the first Phase 2.6.5 batch. +> +> §E's six coverage gaps are all closed — `#W15-16`'s composition test fails by TIMING OUT when the abort +> listener is removed, which is the unkillable run reproduced exactly rather than an assertion standing in +> for it. +> +> **`#W15-2` closed 2026-08-09** as [ADR-0075](../../decisions/0075-fail-closed-resume-on-an-unreadable-event-log.md), +> amending ADR-0074 §5: a read that feeds a REPLAY refuses when any row was skipped, a read that feeds a +> DISPLAY stays tolerant. It landed first on purpose — `#W15-1` adds a new durable event type, and an older +> binary reading it falls straight into §5's skip path, so this decision governs how that event degrades. +> +> **One gap this closing deliberately left open**, named here rather than implied: ADR-0075's refusal is +> pinned at `gate.ts`'s own catch (where the blocker was) and at `toUserFacing`, but NOT through +> `dispatch.ts` / `specs.ts` to the process exit. A re-wrapping `try/catch` added in either layer reproduces +> the same blocker and leaves both tests green. Closing it needs a `run(argv, io)` drive with a redirected +> HOME and a seeded on-disk `history.db`; no such harness exists for `gate` yet, and the marker sits at the +> test it concerns. +> +> **`#W15-1` is what remains, and its DECISION is now made**: +> [ADR-0076](../../decisions/0076-durable-per-attempt-realized-cost-ledger.md) (Accepted 2026-08-09) — an +> additive durable run event recording one settled provider attempt, emitted through ADR-0074 §2's barrier and +> folded into a `run_costs` row at persist time. It landed after ADR-0075 on purpose: a new durable event type +> is exactly the input that decision governs. +> +> **The implementation is staged, and starts here.** In order, because each step is a precondition for the +> next: +> +> 1. `packages/shared/src/run-event.ts` — `cost:attempt_settled`'s schema, its union arm, and +> `parseStoredRunEvent`. Until this lands nothing else compiles. +> 2. `docs/reference/contracts/sse-event-schema.md` — the canonical shape + which of the two cost events is +> authoritative (ADR-0076's last Negative names that drift risk). +> 3. `packages/db/src/run-history-store.ts` — the `applyDerived` arm writing the `run_costs` row in the SAME +> transaction as its event. The `node:completed` delta then telescopes to zero on its own; a test must pin +> that `SUM(run_costs) == runs.total_cost_microcents` still holds, since that is the invariant carrying the +> no-double-count claim. +> 4. The attempt boundary and its barriers — **corrected by +> [ADR-0077](../../decisions/0077-realized-cost-ledger-uses-the-conservative-commitment-barrier.md)**. ADR-0076 +> §1 asked for an `await` at the settle point; there is no `await` to be had there. The seam's observer is +> `onAttempt?: (record) => void` — synchronous, void — and BOTH money events are emitted from that one +> callback a few lines apart, which `budget-governor.ts` already states outright ("which cannot await"). So +> the ledger takes ADR-0074 §2's shape: **start** the write at the settle instant on a chained in-flight +> promise, **join** it at three barriers — before the next egress admission, **before tool dispatch** +> (`agent-turn.ts`'s `await dispatchToolCalls(...)`, the barrier this adds beyond §2), and at the turn/node +> terminal. Every barrier awaits AND observes the failure, because `#emitDurable` is total for store faults +> and resolves. +> 5. `packages/core/src/engine/checkpoint.ts` — **corrected during implementation**. The instruction here was +> "fold it as a SUM of deltas", borrowing ADR-0074 §2's reasoning, and the wrong half of that cost two +> rewrites. Summing into the realized total double-counts (a node terminal's snapshot already contains its +> attempts); summing into a separate accumulator and maxing the two families UNDER-counts (a media node +> writes a snapshot and emits no attempt row, so every attempt after the last boundary vanishes). What is +> correct is the fold already two arms above it: `Math.max` over every durable ABSOLUTE total. §2's +> rejection of `Math.max` does not carry — it was rejected there only because a release can DECREASE a +> conservative total, and realized spend is monotonic. +> +> **STATUS: all five steps are landed** (`8d7ffcf` … `50c60bd`), each with an Opus and a Sonnet round folded. +> The break-verify that matters most was step 4's ordering, and it now holds: with the tests draining to +> quiescence before asserting, deleting B2 reddens the tool-dispatch test and deleting B3 reddens the node- +> terminal test. B1 is masked by B2 on the current fixture (the second egress only comes after the tool), so +> it is covered by construction rather than by a red — named rather than implied. +> +> **This is now the prerequisite of Phase 2.6.5, not a parallel track.** Its five steps touch the same four +> files `CR-10` (ordered append tail) and `CR-12` (effect journal) restructure, so it lands first — see the +> 2.6.5 section below. +> +> **What ADR-0076 explicitly does NOT close**, so it is not read as more than it is: the cost of a TOOL EFFECT. +> A resumed run can still re-execute an `http_request` POST, a `run_command` or an MCP mutation — the +> duplicate EFFECT is the harm, and no cost bookkeeping prevents it. That needs a durable effect journal with +> a prepare/receipt pair, which is a different decision about a different failure and earns its own ADR. It is +> the natural next one after this chain (forward-compatibility → conservative commitment → realized cost). +> +> Two decisions inside the closed set are worth surfacing here rather than leaving in a commit message. +> `#W15-4` **changed a previously-tested behaviour**: #228 pinned that a session survives a failed durable +> write and the next turn is clean; it now stops instead, because continuing to spend above a transcript that +> has fallen behind is the defect. And `#W15-23` is scoped to `run-history-store` only — the other +> `packages/db` stores share the same outer-`db`-inside-a-transaction convention, which remains open, is +> stated here rather than implied closed, and has its own PR queued after this one. + +##### A. Needs an ADR first — the only part that is NOT PR #81's to close + +- ✅ **`#W15-1` (blocker) — a workflow's REALIZED cost is not durable at the provider-attempt boundary.** + *Closed 2026-08-10 by ADR-0076 + ADR-0077 and the five staged steps; the barrier is a chain-and-join, not + the inline await ADR-0076 §1 first specified. See the staged plan below for what each step landed and the + three carried-forward gaps.* + `engine.ts` folds `cost:updated` into memory and streams it; the store documents that it is never persisted, + and the checkpoint can only recover cost from a LATER `node:completed`/gate snapshot. So: a paid call + succeeds → the model asks for a tool → the process dies during the tool → the realized cost is gone and the + resumed node re-runs the same paid call, against a cap that has forgotten the first one. ADR-0074 makes only + the CONSERVATIVE commitment durable; a call that returned trustworthy usage has no equivalent barrier. + *Needs an ADR: an idempotent per-attempt durable ledger, written before the next tool side-effect, the next + egress and the turn/node terminal. Not a local fix — it changes what "spent" means across a crash.* +- ✅ **`#W15-2` (blocker) — a tolerant read is fail-OPEN on the resume path.** ADR-0074 §5 drops an event whose + `type` an older binary does not know, and `checkpointer.ts` builds resumable state from what remains. An + older binary cannot know whether the dropped row was a node terminal, a job submission, a gate decision or a + cost commitment — so it may re-run completed or already-submitted work. Preserving the sequence high-water + mark prevents a UNIQUE collision; it does not restore lost state. Tolerance is right for `logs`/`status` + (read-only); it is wrong for a replay that changes the world. + *Needs a superseding ADR (§5 is append-only): fail closed on the resume path with "a newer Relavium is + required" when any row was skipped, while read-only surfaces stay tolerant — or classify events as + state-bearing vs not, and tolerate only the latter.* + +##### B. PR #81 — ADR-0074 §1, the half that makes the rest safe + +- ✅ **`#W15-3`** (blocker) — the release is exposed but unreachable, and nothing renders `durabilityBroken`. + `GovernorWiring.releaseConservativeCommitments` and `conservativeState()` exist and are tested; no command, + REPL action or Home affordance calls either. §1 requires the surface that RENDERS the amount to also expose + clearing it, and this PR removed the accidental escape a restart used to provide. + *Closed by `/cost --release`: `/cost` renders the durability state, and the release clears the per-model AND + aggregate columns before the live governor, so it survives `chat-resume`. The durable + `budget:estimate_released` named here was NOT needed — the session path restores from columns, not from an + event replay — so it stays reserved; see the lane-(e) note above.* + +##### C. PR #81 — data integrity, each with a concrete failure + +- ✅ **`#W15-4`** (blocker) — session persistence failures are invisible to the producer. `RunEventBus` isolates + listener errors by design, so a persister that cannot write still lets the turn complete. Two consequences: + `persister.ts` advances the in-memory cost total BEFORE the DB write, so a failed `recordSessionCost` leaves + the durable total low and a resume restores a cap that has forgotten real spend; and a failed `commitTurn` + leaves the staged turn uncleared, so the next `beginUserTurn` overwrites the previous user message and the + durable transcript loses an exchange the in-memory session still has. + *Closes when: cost and transcript persistence move to an awaited durability port (UI subscribers may stay + passive), and a persistent write failure puts the session in a durability-degraded state that refuses new + egress rather than reporting success.* +- ✅ **`#W15-5`** (high) — stored payloads are not checked against the authoritative columns. `readEventLog` + selects `seq`/`event_type` alongside the payload but never compares them to the parsed event's own `runId`, + `sequenceNumber` and `type`. A row with `seq = 2` carrying `sequenceNumber: 1` is accepted, leaving the + high-water mark wrong and the resume colliding on the next write. + *Closes when: every denormalized projection is compared after parse and any mismatch is a + `CorruptRunEventError`.* +- ✅ **`#W15-6`** (high) — terminal cost snapshots on failed/cancelled runs are dropped by the read models. + `node:failed`, `run:failed` and `run:cancelled` all carry cost in the shared schema, but the store folds + none of them, the checkpoint folds none, and the run TUI's summary ignores the terminal figure. A sibling's + paid media cleanup after a root failure exists ONLY on `run:failed.cumulativeCostMicrocents`, so replay and + history lose it. Pre-dates this PR, but the PR touches all three files and claims cost safety. +- ✅ **`#W15-7`** (medium) — `media_job:submitted` accepts a half-frozen basis. `units` and + `acceptedCostMicrocents` are independently optional, so a row with a frozen cost and no frozen units resumes + with the old reservation and a re-derived volume. The invariant is not "both or neither" (the approved-bypass + case legitimately has units only) but **`acceptedCostMicrocents` present ⇒ `units` required**. + +##### D. PR #81 — security and correctness + +- ✅ **`#W15-8`** (high) — the new failure-reporting paths can leak credentials. `background-failure.ts` prints an + arbitrary rejection message and, under `RELAVIUM_DEBUG`, a raw stack; `sanitize.ts` strips ANSI/control/bidi + but performs no secret redaction. Same class in the session listener notice, `render-error.ts`'s verbose + stack and `index.ts`'s fatal debug stack. The rejection can be an MCP error, a provider response body or a + tool result — all of which carry keys, `Bearer` headers or credentialed URLs. Existing tests plant ANSI, never + a secret. + *Closes when: the shared secret-shape redactor runs BEFORE terminal sanitization on every one of those paths + (stacks included), with tests planting `sk-*`, `Bearer`, `Basic`, credentialed URLs, query tokens and PEM.* +- ✅ **`#W15-9`** (high) — the stream path can throw out of a contract that says it never throws. + `#runStreamAttempt` calls `#emitSuccess` OUTSIDE its try/catch (the `generate()` path has it inside), so a + provider returning non-integer usage makes `assertAccountableUsage` throw raw out of the async generator — + and `streamOneTurn`'s `for await` has no try/catch, so it crashes the turn unclassified, AFTER the content was + produced and paid for. +- ✅ **`#W15-10`** (high) — `--json` output is not sanitized. `status --json` / `gate list --json` carry + model-controlled text; `JSON.stringify` does not escape Unicode bidi overrides (CVE-2021-42574). The human + render path is sanitized; the JSON branch is not. `createJsonRenderer` is the same gap on the NDJSON path. +- ✅ **`#W15-11`** (high) — `createPlainRenderer` sanitizes AFTER splitting on newlines, so an embedded newline + still forges a line. Every other boundary this PR touched sanitizes first; `renderer.ts` is the exception. +- ✅ **`#W15-12`** (high) — `isBlankPreview` treats a fully-redacted preview as informative (`chat-mode.ts`), so a + user pressing "always" on `Approve write to [redacted]?` auto-approves every `write_file`/`run_command`/ + `http_request` for the session, having been shown nothing. +- ✅ **`#W15-13`** (medium) — the config reader chmods PROJECT configs to 0600. `reassertOwnerOnly` runs on every + layer, but `config-spec.md` says project files are committed and shared. Reading a config mutates repo file + permissions, and `statSync`/`chmodSync` follow symlinks. Self-heal belongs to the canonical global config + only. +- ✅ **`#W15-14`** (medium) — a provider `Retry-After` sleep is not abort-aware. `fallback-chain.ts` passes the + delay straight to an injected `sleep` with no signal, and the real hosts use a plain `setTimeout`. A cancel + during a 60 s clamped wait hangs until the timer fires. The existing abort test resolves `sleep` immediately, + so the window is untested. The docblock also says an oversized value becomes `undefined`; the code clamps. +- ✅ **`#W15-15`** (medium) — corruption is reported to a multi-run view as "no data". `readPerRunOrDegrade` + returns only the fallback, so `gate list` can print "No pending human gates." for a run whose gates were lost + to corruption. It must return the degradation alongside the value, with a warning in human output and a + machine-readable marker under `--json`. + +##### E. PR #81 — coverage gaps it shipped knowingly + +All six are now CLOSED and their in-source markers removed. None had been a shipped defect — each had +mutation-verified coverage one layer down — but closing them surfaced one that was: `onLegacyMediaJobHold` +reached `WorkflowEngine` from a real `gate.ts` sentence and the constructor never read it, so ADR-0074 §3's +hold notice was dead and a `relavium gate` resume of a legacy parked media job stalled silently. Found by +the review of `#W15-16`, fixed with it, and now the signal that test drives. + +- ✅ **`#W15-16`** — the engine-level composition test for §3's abort-breakable media-job hold; deleting the abort + listener leaves the whole suite green. +- ✅ **`#W15-17`** — the CLI-layer test that `restoreConservativeCost` is wired through `buildSessionRuntime`. +- ✅ **`#W15-18`** — the `coalesce` backfill of `session_costs.model_catalog_id` (needs a `model_catalog` FK row + the block does not seed). +- ✅ **`#W15-19`** — that a chat turn's terminal is emitted only AFTER `flushBudgetCommitments` resolves. +- ✅ **`#W15-20`** — the H3 approved-bypass `acceptedCostMicrocents` omission; reverting it leaves the suite green. +- ✅ **`#W15-21`** — `agent-run.ts`'s `attachConservativeWriter`; deleting it silently breaks every capped + `agent run`. + +##### F. PR #81 — standards and documentation drift + +- ✅ **`#W15-22`** (low) — the PR's "no unsafe `as`" conformance claim is false. Two assertions in new production + code: `shared.ts`'s `readRetryAfter` casts an arbitrary `headers.get`, and `openai.ts`'s `mapOpenAiApiError` + casts a `headers` field back onto a parameter type that lacks it. A structural `headers?:` plus runtime + narrowing removes both. +- ✅ **`#W15-23`** (medium) — `persistEvent`'s `applyDerived` uses the outer `db`, not the transaction handle. + Harmless under `better-sqlite3`, but it breaks ADR-0005's Postgres-portability promise on the highest-volume + money write. +- ✅ **`#W15-24`** (low) — canonical docs describe code that was not shipped. `tool-registry.md` says path + redaction is per-segment (it is whole-string, and the same file says so correctly three paragraphs later); + `llm-provider-seam.md` still claims model-discovery/media-poll keep a small SDK retry; ADR-0028's amendment + note still says ADR-0074 §2–§5 have not landed. Each could send a future reader back toward a #91-class hole. diff --git a/docs/roadmap/phases/phase-2.6-conversational-authoring.md b/docs/roadmap/phases/phase-2.6-conversational-authoring.md index 340b344b..e8e69af1 100644 --- a/docs/roadmap/phases/phase-2.6-conversational-authoring.md +++ b/docs/roadmap/phases/phase-2.6-conversational-authoring.md @@ -3,6 +3,9 @@ > Status: **In progress** — **2.6.F** (platform floor + full-screen TUI, PR #74) and **2.6.C** (reseat > transcript-carry + `/cost`, PR #75) are ✅ **Done**, and **2.6.Q P1–P5** landed with PR #76 (2026-07-26). > Joint Wave 2 prerequisites and corrections merged in PR #94 (2026-10-15); the [merge checkpoint](phase-2.5.5-hardening-and-remediation.md#wave-2-merge-checkpoint--2026-10-15) records their exact scope. Full M/N lifecycle and remaining workstreams stay open. +> **CLI MVP first (2026-11-04):** the MVP order in [../current.md](../current.md#next-work--the-cli-mvp) takes slices of +> 2.6.A/B/D/E/G/H/I/J/K/M/Q ahead of the rest of the phase; new MVP items are defined under +> [CLI MVP additions](#cli-mvp-additions--2026-11-04). The phase scope, exit criteria and milestones are unchanged. > Live progress is canonical in [../current.md](../current.md); the rest of the phase follows. Depends on the Phase 2.5 > spine (the wired tool-environment and the per-tool approval / mode system), which is **complete** (M2.5-4, > PR #69, 2026-07-08), so this phase is unblocked. @@ -314,6 +317,45 @@ viewport with an inline switch marker (regression-tested on both the Home and st shows per-model spend for a reseated session; the notice is asserted on resume surfaces. **Required ADR:** none (ADR-0059 is Accepted). +#### Implementation record (moved from current.md on 2026-11-04) + +Driven by two maintainer manual-test findings. **F1:** a mid-session `/models` reseat **blanked the alt-screen +viewport** — the reseat builds a fresh view store, and on the full-screen renderer (the TTY default since 2.6.F, with +no native scrollback behind it) that store *is* the scrollback, so the whole conversation vanished the moment a user +switched models. The durable transcript and the new model's context were never at risk; the loss was the view. The +reseat now carries the rendered conversation, gated to full-screen (a seeded **inline** store would re-print the +conversation over the copy ink's `` already put in the terminal's scrollback), and the switch notice lands +**beneath** it as an inline `⇄ model changed → ` marker, keeping ADR-0059's bound disclosure clause. + +**`/cost`** now shows where the money went, behind a new +[ADR-0070](../../decisions/0070-durable-per-model-session-cost-attribution.md): a durable `session_costs` table with the +invariant **`SUM(rows) == agent_sessions.total_cost_microcents`**, true by construction. ADR-0059 had planned to +derive the breakdown from the per-message `model_id`, which **cannot work** — one row holds one model, but a turn +whose tool loop fails over bills two. Along the way it closed a **live bug**: `total_cost_microcents` was being SET +blindly by five writers, so two `chat-resume` processes could permanently corrupt a session's total. It now has a +single owner. `session_messages`' never-written token/cost columns were dropped rather than left as a second, wrong, +zero-valued source. + +Four adversarial review rounds ran over the work and each found something real — including that a tripwire I shipped +was **armed in production** and **swallowed by ink** (so it could never fail), that a perf harness measured a +**pointer assignment** while its ADR note claimed it verified an O(n) bound, and that the `/cost` DB read — the +step's central change — had **zero test coverage** at any layer. All folded, each with a break-verified regression. + +**Open obligations carried out of 2.6.C:** + +- **`chat-resume` opens on an empty viewport** — it has never repainted prior turns, in any mode or version (nothing + projects `session_messages` into rendered entries). Not a 2.6.F regression; tracked in + [deferred-tasks.md](../deferred-tasks.md). +- **Reported spend is a systematic under-estimate of the provider's invoice** — an egress that streamed content but + ended without a usage chunk, and a mid-stream failure, are recorded as 0 on *both* sides of the invariant. That is + a usage-capture gap in the seam, filed against **2.6.Q** (ADR-0070 §3). +- **Cross-turn tool-call memory** — a "the reseat forgets tool calls" report turned out to be an *engine* property, + not a reseat one: tool pairs never cross **any** turn boundary (ADR-0062 §6). Deferred behind a default-off toggle, + but **three already-live bugs gate it** (no pre-egress context-window check; overflow is a fatal, non-retryable + `bad_request`; the budget gate prices only output tokens) — see [deferred-tasks.md](../deferred-tasks.md). +- **The CLI e2e suite migrates the developer's real `~/.relavium/history.db`** — verified, and it is what let a + broken migration reach real data while staying invisible to CI. Tracked in [deferred-tasks.md](../deferred-tasks.md). + ### 2.6.D — Session `{{ctx.*}}` prompt interpolation Unchanged in intent; one absorbed deferred item makes the security mandate explicit. @@ -392,7 +434,8 @@ tests. **Required ADR:** `ink-syntax-highlight` dependency + markdown rendering > [ADR-0069](../../decisions/0069-string-width-for-the-cli-renderer.md) (`string-width`, Accepted on the merge). > The shipped inventory (incl. the `✓ Copied` copy-on-select toast), the two adversarial review rounds, and the > open obligations carried out (short-terminal Home overflow → 2.6.G; `relavium run` stays inline; themes → 2.6.L) -> live in the canonical status home [current.md](../current.md); this entry is not restated there. +> are in the implementation record at the end of this section, moved from current.md on 2026-11-04. Its release +> clause (v0.2.0) is still open: it lands with the CLI MVP's MVP-10 ([current.md](../current.md#next-work--the-cli-mvp)). The substrate workstream — it runs **first** because the browsers (2.6.G), settings screens (2.6.L), and render-v2 (2.6.M) all build on it. @@ -442,6 +485,45 @@ and released as a breaking 0.x MINOR bump (e.g. `0.2.0`, per ADR-0067) with migr frozen-clock regression pinned. **Required ADRs:** the ADR-0021-superseding floor ADR + a full-screen renderer & TUI-harness ADR. +#### Implementation record (moved from current.md on 2026-11-04) + +The first 2.6 workstream. Behind [ADR-0067](../../decisions/0067-node-supported-floor-22-reaffirm-better-sqlite3.md) +(Node `>=22` published floor, `>=22.13.0` dev-install floor; `better-sqlite3` re-affirmed), +[ADR-0068](../../decisions/0068-full-screen-tui-renderer-ink7-harness.md) (`ink` 7 + a hand-built alternate-screen +renderer + the `ink-testing-library` harness), and +[ADR-0069](../../decisions/0069-string-width-for-the-cli-renderer.md) (`string-width` for display width — **Accepted** +on the PR #74 merge, the CLAUDE.md rule-2 approval it gated). + +Shipped: `ink` 6→7 and the component-render harness; the alt-screen lifecycle with restore nets on every +termination path; a hand-built transcript **viewport** (grapheme-aware wrapping, windowing, scroll + auto-follow, a +per-entry wrap cache) and the inter-session alt-buffer **hoist**; the **`/scrollback`**, **`/edit`** and **`/copy`** +copy-and-search hatches over a `suspendFullScreen` primitive; **in-app mouse text selection with copy-on-select over +OSC 52** on both surfaces, with edge auto-scroll, a frozen auto-follow and a transient **`✓ Copied` toast** so the +otherwise-silent copy is confirmed; `--no-mouse` / +`[preferences].{alt_screen,mouse,copy_on_select,show_banner}`; a branded Home banner; and DEC-2026 synchronized +output — which `ink` 7 turned out to already emit, so the step **pins** it rather than building the planned +`stdout` Proxy. + +The full-screen renderer is now the **default on a TTY**; `--no-alt-screen` and a machine / non-TTY / `--json` / CI +path stay on the byte-identical inline renderer. + +Two adversarial review rounds (one per step, one over the whole phase) drove the last third of the work. The +whole-phase round found that **the caps-lift ADR-0068 exists to deliver was never implemented** — a name collision +with the Step-4b-3 wrap cache — so a >4000-character answer was still clipped in the very viewport built to scroll +it; that `displayWidth` under-counted 8 539 code points against `ink`'s own width function; and that a keyboard +Ctrl-C during a Home hatch stranded mouse reporting on the user's shell. All folded, each with a break-verified +regression test. + +**Open obligations carried out of 2.6.F:** + +- **The Home landing can overflow a short terminal** — a populated "Attention required" section pushes the top off, + and the alt buffer has no scrollback to recover it. Predates 2.6.F (the strip is 2.5.B); **2.6.G**'s management + browsers replace the strip wholesale, which is the right place to fix it. +- **`relavium run`'s TUI stays inline** (a deliberate 2.6.F scope cut); its retained-scrollable-history is tracked. +- **Themes** (beyond colour / `NO_COLOR`) are deferred to **2.6.L**, per the maintainer's Step-5 decision. +- Real-TTY signal paths (double-Ctrl-C, `kill -TERM`/`-HUP`/`-QUIT`, a hatch under `tmux`) remain a **manual PR-time + check** — the unit tests pin the orchestration around injected seams, not the kernel. + ### 2.6.G — Home management browsers: workflows, runs, agents The interactive browsers that make the Home the management center. The interaction model follows the @@ -928,6 +1010,9 @@ sensitive-read floors). wired for chat in 2.6.N — agent spawning is separate from workflow composition.) - **Default chat agent grant review**: widen the built-in agent's grant to the new idempotent read tools (search/find/todo); write/exec/egress stay opt-in via mode + approval. + *Direction set 2026-11-04 (D62, see [MVP-A14](#cli-mvp-additions--2026-11-04)):* the built-in agent also gets + `write_file`/`edit_file` and `run_command` under the approval card; this lands only with its ADR-0057 amendment and + security review. - **Curl/wget as web-search substrate** *(research item)*: evaluate whether `curl` / `wget` / `httpie` / `xh` should be **default-allowed** in the built-in chat agent's `allowed_commands` as an alternative web-search path — the model can already use `web_search` (once credential-resolved), but a @@ -1237,6 +1322,40 @@ extends ADR-0064; rides ADR-0065 (SSRF / user-pricing precedent) and ADR-0066 (t [ADR-0011](../../decisions/0011-internal-llm-abstraction.md)-respecting (host-side data source, not a `listModels` call). Mandatory security review is a co-gate. +## CLI MVP additions — 2026-11-04 + +The CLI MVP (decision D58, milestone M3.5, v0.2.0) is ordered in [current.md](../current.md#next-work--the-cli-mvp). +Most of its items already have a home in a workstream above. This section is the one home for the items that had +none, found by the 2026-11-04 review of the code at `5a328bca` and ruled by the maintainer +([priority reset](phase-2.5.5-hardening-and-remediation.md#mvp-priority-reset--2026-11-04)). Each belongs to the +workstream named in its row and follows that workstream's ADR and security gates. + +| ID | Workstream | Item and acceptance | +|---|---|---| +| MVP-A1 | 2.6.G | **Human run output.** `relavium run` and `logs` print the workflow's outputs on the human surfaces, bounded and through the shared sanitize floor; `--json` is unchanged. | +| MVP-A2 | 2.6.G | **The run id is visible.** The TTY run view and its final summary show the run id that `logs`, `status` and `gate` accept. | +| MVP-A3 | 2.6.B | **`/export` writes where workflows live.** It writes into `.relavium/workflows/` with a slug from the session title (no UUID file in the working directory) and offers to run the result. | +| MVP-A4 | 2.6.J | **The default model follows the stored key.** With no configured default, the chat model comes from a provider that has a key, and a missing key yields the actionable "no API key … `provider set-key`" error instead of an opaque credential failure. | +| MVP-A5 | 2.6.J | **`relavium chat` offers onboarding** when no key exists, without re-running the wizard on every launch when the keychain is unavailable (#55's loop stays fixed in its own row). | +| MVP-A6 | 2.6.J | **Standard provider environment variables** (`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, `DEEPSEEK_API_KEY`) are read after the `RELAVIUM_*` variables, never logged or echoed; [config-spec.md](../../reference/contracts/config-spec.md) is updated with the code. Key-custody review. | +| MVP-A7 | 2.6.Q | **Interactive output budget.** No implicit 4,096-token cap on interactive Anthropic chat: the default is the catalog ceiling, clamped, with a `[chat].max_tokens` key; the budget estimate's interplay (ADR-0101) is checked. | +| MVP-A8 | 2.6.M | **Truthful tool rows.** A failed tool call shows a failure mark, not ✓, and the assistant's text between tool calls stays visible. | +| MVP-A9 | 2.6.M | **Ranged reads.** `read_file` takes line or byte ranges and bounded output spills to a wired `ToolHost.outputStore`; its model-visible description matches the CLI's binary refusal. | +| MVP-A10 | 2.6.M | **Denials reach the model.** A user or guardrail denial returns to the model as a `tool_result` instead of ending the turn (ADR-0057 amendment, D67). | +| MVP-A11 | 2.6.M | **Configurable, finite ceilings.** Tool rounds, correction attempts and the session turn limit are configurable with conservative defaults and stay finite (ADR-0086). | +| MVP-A12 | 2.6.E | **One keymap.** Ctrl+C interrupts the running turn, a second Ctrl+C ends the session, and Ctrl+C at a pending approval rejects it before anything else (keymap ADR, D68; supersedes the 2026-10-19 `/cancel` wording; approval-path sitting). | +| MVP-A13 | 2.6.M | **`[chat].default_mode`.** The starting mode is configurable (default `ask`), with a one-time hint that Shift+Tab changes it (ADR-0057 amendment, D62). | +| MVP-A14 | 2.6.M | **Default grant (D62).** The built-in agent gets the read, search and find tools, `write_file` and `edit_file` (hidden in ask/plan, per-call diff approval in accept-edits) and `run_command` under MVP-A15; egress stays opt-in. Lands only after the approval diff; ADR-0057 amendment and security review. | +| MVP-A15 | 2.6.G | **`run_command` approval card (D63).** A non-allowlisted command prompts once / for the session / to remember; remembered grants live outside the repository, in operator-owned configuration keyed by project root; accept-edits keeps prompting for allowlisted commands; headless paths stay fail-closed; `!` uses the same card. New ADR amending ADR-0029(a)/0061/0063, and a security sitting. | +| MVP-A16 | 2.6.D | **Project instructions.** A project instructions file (`AGENTS.md`) and an environment block enter the system prompt under explicit provenance rules (new ADR; independent of ADR-0060's `{{ctx.*}}`). | +| MVP-A17 | 2.6.E | **Session entry flags.** `relavium chat --model/--provider` and a one-shot prompt mode; headless stays fail-closed; both follow ADR-0110's compatibility rules. | +| MVP-A18 | 2.6.B | **A minimal `relavium init`** scaffolds `.relavium/` instead of being a dead command. | +| MVP-A19 | 2.6.B | **Starter content.** Two or three starter workflows and agents a new user can run unchanged. | +| MVP-A20 | 2.6.I | **Masked `provider set-key` on a TTY** instead of refusing an interactive terminal. Secret-input sitting. | +| MVP-A21 | 2.6.I | **Registered MCP servers reach the default agent**, under ADR-0084 spawn consent and ADR-0088's untrusted tool definitions. | +| MVP-A22 | 2.6.F | **Upgrade from `0.1.1`.** A populated `0.1.1` home (history.db with migrations 0000–0004, config.toml, keychain entries) upgrades through v0.2.0, with a seeded-then-migrated test. | +| MVP-A23 | 2.6.F | **Release prerequisites for v0.2.0.** The D59 release ADR and guard change; version and CHANGELOG including the Node ≥22 break; npm token, 2FA, provenance and the `v*` ruleset verified; the interactive Home and chat checked on a real Windows terminal; a manual three-provider end-to-end turn; first-read docs (README, npm README, tutorials, runbooks) describing what ships. | + ## Deferred-tasks pulled into this phase The [deferred-tasks.md](../deferred-tasks.md) triage (2026-07-08) mapped these confirmed-open items into @@ -1303,7 +1422,7 @@ separate backlog. | In-phase | Completed by | Outcome | |----------|--------------|---------| -| M2.6-1 Foundation | 2.6.F | Node floor decided/shipped; full-screen renderer + TUI harness — the substrate for every arm | +| M2.6-1 Foundation | 2.6.F | Node floor decided/shipped; full-screen renderer + TUI harness — the substrate for every arm *(met except the v0.2.0 release clause, which closes with the CLI MVP's MVP-10)* | | M2.6-2 Authoring core shared | 2.6.A | `@relavium/authoring` + catalog-aware pre-flight | | M2.6-3 Conversational authoring | 2.6.B + 2.6.D | "define a workflow…" → valid YAML in the Home; `{{ctx.*}}` lands | | M2.6-4 The Home-managed CLI | 2.6.G + 2.6.H + 2.6.I + 2.6.J + 2.6.K | Every management task doable from Home; drillable, attributed run history; onboarding v2 | @@ -1511,9 +1630,9 @@ flowchart LR | 6 | *(new)* | Management browsers + durable run detail (amends ADR-0036) | Drafted when 2.6.G starts | 2.6.G / 2.6.H | | 7 | *(new)* | MCP management surface + config-write extension (extends ADR-0063) | Drafted when 2.6.I starts | 2.6.I | | 8 | *(new)* | Onboarding auth paths + Relavium-account forward design (rides ADR-0012–0015) | Drafted when 2.6.J starts | 2.6.J | -| 9 | *(new)* | Toolbelt additions + tool-render/approval-preview contract + the `ask_user` mid-turn pause/resume **engine** interaction (rides the EA4 pause machine; a turn-level pause distinct from the approval pause) (extends ADR-0029/ADR-0057) | Drafted when 2.6.M starts | 2.6.M | +| 9 | *(new)* | Toolbelt additions + tool-render/approval-preview contract + the `ask_user` mid-turn pause/resume **engine** interaction (rides the EA4 pause machine; a turn-level pause distinct from the approval pause) (extends ADR-0029/ADR-0057) | Drafted when 2.6.M starts *Split 2026-11-04 for the MVP: the toolbelt and the tool-render/approval-preview contract first (MVP-3), the `ask_user` pause later.* | 2.6.M | | 10 | *(new)* | i18n + theming architecture | Drafted when 2.6.L starts | 2.6.L | -| 11 | *(new)* | Syntax highlighting dependency (`highlight.js` + `cli-highlight` via `ink-syntax-highlight`) + markdown rendering architecture | Drafted when 2.6.E starts | 2.6.E | +| 11 | *(new)* | Syntax highlighting dependency (`highlight.js` + `cli-highlight` via `ink-syntax-highlight`) + markdown rendering architecture | Drafted when 2.6.E starts *Split 2026-11-04 for the MVP: part 1 is an in-house markdown/code-block renderer with no dependency (MVP-2); the highlighter decision follows the MVP.* | 2.6.E | | 12 | *(new)* | Child-session **foundation** (extends ADR-0024/ADR-0036; parent-child lineage schema, standardized I/O contract, the **host artifact-store port** + central ephemeral root + fs-floor home-anchoring, catalog spawn, cost roll-up, abort propagation, depth/concurrency guardrails) | Drafted when 2.6.N starts | 2.6.N | | 13 | *(new)* | On-the-fly **generation + parallel orchestration security** (the generation-as-codegen threat model, schema-valid≠safe, tool-grant clamping, the parallel error-policy contract; sits on ADR-0060's taint precedent) — **mandatory security review is the ship-gate** | Drafted when 2.6.O starts | 2.6.O | | 14 | *(new)* | `subworkflow` node + **`schema_version` 1.1** additive migration (the first versioned-schema bump — additive/opt-in, subworkflow-only not `loop`; the nested-run event namespace; the recorded deferral of dynamic `invoke_workflow`) | Drafted when 2.6.P starts | 2.6.P | diff --git a/docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md b/docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md index 6169cff2..1fad5b12 100644 --- a/docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md +++ b/docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md @@ -1553,6 +1553,10 @@ Every connect failure had collapsed into one sentence, and every identifier was carried — the package's own tests had to regex human prose to find a server id. Fields and a `reason` now, derived from the caught error's SHAPE, never from its text. +### W4 start gate — maintainer ruling (moved from current.md on 2026-11-04) + +**The `W4`-waits-for-the-`W3`-blocker gate was lifted deliberately on 2026-09-01, by the maintainer, and it is recorded here rather than left to be inferred from the commit order.** The reasoning: the blocker is a `BoundedEventStream` drop on a stream nobody iterates, its fix is ADR-0087 §1 — a decision that was still **Proposed** and unapproved — and `W4` touches a disjoint surface (the MCP boundary). Blocking a whole wave on an unapproved ADR would have stalled the phase rather than protected it. **The blocker stays open and stays first in the `W3` residuals**; nothing here closes it or reduces its severity. **Updated 2026-09-04:** ADR-0087 is now **Accepted**, with nine dated corrections written into it after a review refused acceptance. That removes the "unapproved ADR" half of this gate — §1 is now a settled decision — but §1 remains **unimplemented** and the blocker is unchanged. Its Correction 1 records what §1 must settle before it can be built: the two factories §1 names are not symmetric — a host calls `createSessionHandle` directly, but a `RunHandle` is only ever produced inside `WorkflowEngine.start()` / `resumeFromCheckpoint()`, so the run half needs the mode plumbed onto an engine entry point before it can be passed at all. A `subscribe-only` handle must also not present a silently-empty iterable. + ### W4 closing register **The register, per exit criterion 7 — the code that closes each item, so a reader can check the claim.** From 0826f30a22e12f106cdd684a9d4d840270710801 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Wed, 4 Nov 2026 13:56:27 +0300 Subject: [PATCH 89/97] fix(core): mask credential flags and URL passwords, bound omitted media Three verified PR #96 review findings in the journal and display projection: - A flag whose last word names a credential (--auth-token, --access-token, --pwd, --x-api-key, --credentials) now masks its argument, as the object-key rule already masked those names as keys; curl's -u/--user user:password pair is masked in every spelling. A flag that only mentions a credential (--password-stdin, --token-file, --max-tokens) or starts with no- keeps the next argument's identity. - A `,` or `/` in a URL password no longer ends the authority: the userinfo is read on to its `@`, unless a port, a public query contact, another scheme or a JSON/quote delimiter comes first. The leaking test pins are replaced; quotes, `}` and `]` stay documented differences. - boundForModel counts the inline media the texts omit (data URIs, base64 sources, binary buffers) against the model-facing ceiling. A 30 MiB data URI was measured as its marker, passed whole to the model and settled into history.db; the spill and preview stay media-free. Also clears Sonar's reduce-without-initial-value bugs (S6959, S7727) and two smells in the escape view. The secrets controls' pinned case counts follow the new tests (234 bounding, 116 registry); their unnecessary String.raw literals are plain strings with identical values. A manual mutation sweep killed all 24 mutants. Affected v5 journal rows refuse matching prepare, as the PR95 tightening documented. Refs: ADR-0080, ADR-0089 Co-Authored-By: Claude Opus 5.5 --- docs/reference/shared-core/built-in-tools.md | 26 +- docs/reference/shared-core/effect-journal.md | 6 +- docs/reference/shared-core/tool-registry.md | 6 + packages/core/src/tools/bounding.test.ts | 131 +++++++++- packages/core/src/tools/bounding.ts | 249 +++++++++++++++---- packages/core/src/tools/registry.test.ts | 23 ++ tools/causal-controls/provenance.test.mjs | 4 +- tools/secrets-controls/README.md | 4 +- tools/secrets-controls/mutations.mjs | 36 +-- 9 files changed, 412 insertions(+), 73 deletions(-) diff --git a/docs/reference/shared-core/built-in-tools.md b/docs/reference/shared-core/built-in-tools.md index 6af235a2..8dc999ad 100644 --- a/docs/reference/shared-core/built-in-tools.md +++ b/docs/reference/shared-core/built-in-tools.md @@ -97,7 +97,7 @@ A raw `?` or `#` before a URL authority's `@` begins public query or fragment te text before it is host-shaped (a dotted name or IPv4, `localhost` or a bracketed IP literal, with an optional port) **and** an `=` assignment follows it; otherwise the userinfo is masked. This is the same predicate as the -[LLM diagnostic backstop](../../standards/error-handling.md#diagnostic-secret-backstop), including +[LLM diagnostic backstop](diagnostic-secret-backstop.md), including its 260-character bound on the host-shaped prefix. Masked userinfo that holds such a raw `?` or `#` leaves the authority ambiguous: the userinfo may also be a query that another rule reads (`https://gateway?password=P@ss.word1`). The backstop masks through the authority's end unless a @@ -121,11 +121,25 @@ user, so its value does not run on through the public host and path (`https://x-access-token:@github.com/o/r` → `https://[redacted]@github.com/o/r`), and a key on the line after a split URL masks its value to its own end (`see https://\nDB_PASSWORD=P@ssw0rd123 now` → `see https://\n[redacted] now`). The projection -otherwise differs from that backstop by design: its authority also ends at `"`, `'`, `,`, `}` and -`]`, so userinfo that contains one of them — an already-masked `[REDACTED]` password among them — is -left as it is; it recognizes any scheme of up to 32 characters rather than a finite list; and a -userinfo with no password is kept as a vanity handle below 16 characters and masked from 16 on, -under every scheme. +otherwise differs from that backstop by design: its authority also ends at `"`, `'`, `}` and `]`, +the delimiters of the JSON and quoted texts it reads, so userinfo that contains one of them — an +already-masked `[REDACTED]` password among them — is left as it is; it recognizes any scheme of up to +32 characters rather than a finite list; and a userinfo with no password is kept as a vanity handle +below 16 characters and masked from 16 on, under every scheme. Since 2026-11-04 a `,` or `/` in a +password no longer ends the authority: the userinfo is read on to the first `@` within 512 characters +(`https://user:pa,ss@host` and `https://user:pa/ss@host` → `https://[redacted]@host`). A port +(`https://host:8080/a@b`, `https://host:8080,bob@x.example`), a public query contact as above, another +scheme, or a quote, `}`, `]` or whitespace first keeps the text public; the backstop still stops at a +`/` (a recorded low). + +In an argument vector, the element after a credential flag, or the value of its `--flag=value` +spelling, is masked whole. A flag is a credential flag when it is one of the anchored names (`-p`, +`--password`, `--token`, `--api-key`, …) or its last word names a credential, read with the object-key +rule's words as written (`--auth-token`, `--access-token`, `--db-password`, `--pwd`, `--x-api-key`, +`--credentials`). A flag that only mentions a credential (`--password-stdin`, `--token-file`, +`--max-tokens`) or begins with `no-` does not count, so the argument after it keeps its own identity. +Only `-p` keeps a port exception. curl's `-u`/`--user` value, glued (`-ualice:secret`) or not, is +masked when it holds a `:`, unless it is a `uid[:gid]` or a URL. LF/TAB inside a recognized token-shaped credential extends the masked original span. An ambiguous token-alphabet continuation after LF/TAB is conservatively masked with that credential; an diff --git a/docs/reference/shared-core/effect-journal.md b/docs/reference/shared-core/effect-journal.md index da05fcfd..474db201 100644 --- a/docs/reference/shared-core/effect-journal.md +++ b/docs/reference/shared-core/effect-journal.md @@ -422,7 +422,11 @@ or Token scheme survives as its canonical label followed by `[redacted]`; no ori credential remainder or casing is retained. Credential flags in an argument vector mask the whole following element or the complete value in `--token=value` form. Only the ambiguous `-p` spelling retains the existing port-shaped exception; explicitly named -credential flags have none. Unrecognized public fields and ordinary arguments still +credential flags have none. Since 2026-11-04 the credential flags include every flag whose last +word names a credential and curl's `-u`/`--user` pair, and a URL password's `,` or `/` no longer +ends its userinfo ([built-in tools](built-in-tools.md#finite-diagnostic-credential-masking)); as +with the PR95 tightening above, a `v5:` row whose arguments these project differently now refuses +matching prepare, and every other row keeps its digest. Unrecognized public fields and ordinary arguments still contribute their complete surviving values to identity. These projections do not change the arguments dispatched to the host. diff --git a/docs/reference/shared-core/tool-registry.md b/docs/reference/shared-core/tool-registry.md index 97bf1e13..c3c24049 100644 --- a/docs/reference/shared-core/tool-registry.md +++ b/docs/reference/shared-core/tool-registry.md @@ -418,6 +418,12 @@ distinct from the *event* `outputSummary`, which the registry bounding stage pro The ceiling is a **byte/line** bound (no token count — that needs a provider-specific tokenizer, which would break engine purity); concrete defaults and the marker shape are canonical in [built-in-tools.md](built-in-tools.md#tool-result-bounding). `ToolResultLimits` carries them into `ctx`. +The bound counts the inline media a result still holds — a base64 `data:` URI, a canonical base64 +source, a binary buffer — at its full size, although the summary, spill and preview show only a +marker for it: a builtin's media rides the attachment rail ([ADR-0089](../../decisions/0089-media-correctness-four-boundaries.md) §1), +so media left in a dispatch result is text to the model and is settled into the journal. Over the +ceiling, the model gets the media-free preview and the marker reports the full byte count (2026-11-04; +a 30 MiB data URI had been measured as its marker). Observability and journal-argument projections distinguish shared siblings from ancestor cycles. Each walk removes a visited object when its branch finishes, including native/binary/media early diff --git a/packages/core/src/tools/bounding.test.ts b/packages/core/src/tools/bounding.test.ts index 6a7e4559..bb2d59d2 100644 --- a/packages/core/src/tools/bounding.test.ts +++ b/packages/core/src/tools/bounding.test.ts @@ -288,6 +288,40 @@ describe('boundForModel', () => { expect(bounded.summary).toContain('omitted'); }); + it('counts a 30 MiB data URI the texts omit against the model-facing ceiling', async () => { + // A PR #96 review: the ceiling was measured on the 25-character marker, so the whole URI went to the + // model and was settled into the effect journal. The spill and the preview stay byte-free (I3). + const uri = `data:image/png;base64,${'A'.repeat(30 * 1024 * 1024)}`; + let spilledText = 'unset'; + const spill = vi.fn((text: string) => { + spilledText = text; + return Promise.resolve({ ref: 'spill://uri', byteLength: text.length }); + }); + const bounded = await boundForModel(uri, BIG, host({ outputStore: { spill } })); + expect(bounded.truncated).toBe(true); + expect(String(bounded.value)).not.toContain('AAAAAAAA'); + expect(String(bounded.value)).toBe( + `[base64 data URI omitted]\n\n[… truncated: ${utf8ByteLength(uri) + 25} bytes / 1 lines; full output at spill://uri …]`, + ); + expect(spilledText).toBe('[base64 data URI omitted]'); + expect(bounded.summary).not.toContain('AAAAAAAA'); + }); + + it('counts base64 sources and binary buffers inside a result, and keeps one that fits', async () => { + const limits: ToolResultLimits = { maxBytes: 1024, maxLines: 100 }; + const fits = { source: { kind: 'base64', data: 'Q'.repeat(512) } }; + expect((await boundForModel(fits, limits, host())).value).toBe(fits); + for (const over of [ + { source: { kind: 'base64', data: 'Q'.repeat(2048) } }, + { thumb: new Uint8Array(2048) }, + [{ uri: `data:audio/mpeg;base64,${'Q'.repeat(2048)}` }], + ]) { + const bounded = await boundForModel(over, limits, host()); + expect(bounded.truncated, JSON.stringify(Object.keys(over))).toBe(true); + expect(String(bounded.value)).not.toContain('QQQQ'); + } + }); + it('redacts base64 from EVERY element of an array-of-media result (the redaction walk recurses arrays)', async () => { const a = 'aGVsbG8gZmlyc3QgbWVkaWEgcGFydCBwYXlsb2Fk'; const b = 'd29ybGQgc2Vjb25kIG1lZGlhIHBhcnQgcGF5bG9hZA=='; @@ -851,6 +885,66 @@ describe('redactSecretShapedValue — the KEY decides, whatever the value looks }); }); + it.each([ + '--auth-token', + '--access-token', + '--pwd', + '--db-password', + '--x-api-key', + '--apiKey', + '--credentials', + '--refresh_token', + ])( + 'STRUCTURAL_ARGV_WORDS masks the value after %s, whose last word names a credential', + (flag) => { + // A PR #96 review: the object-key rule masked these names as keys while the anchored flag list passed + // them verbatim into `agent:tool_call` and the journal projection. + expect(redactSecretShapedValue({ args: [flag, 'Qz7flagValue', '--count=4'] })).toEqual({ + args: [flag, '[redacted]', '--count=4'], + }); + expect(redactSecretShapedValue({ args: [`${flag}=Qz7flagValue`] })).toEqual({ + args: [`${flag}=[redacted]`], + }); + expect(projectSecretShapedValue({ args: [flag, 'Qz7flagValue'] })).toEqual({ + value: { args: [flag, '[redacted]'] }, + settled: true, + }); + }, + ); + + it('STRUCTURAL_ARGV_WORDS leaves a flag that only mentions a credential, and its next argument', () => { + // Masking these would give calls that differ only in the next argument one journal identity. + const benign = { + args: [ + ...['--max-tokens', '4096', '--password-stdin', 'next', '--token-file', '/tmp/t'], + ...['--no-password', '-h', 'db', '--key', 'k3', '--author', 'me', '--token-type', 'bearer'], + ], + }; + expect(redactSecretShapedValue(benign)).toEqual(benign); + }); + + it('STRUCTURAL_ARGV_USERINFO masks a curl user:password in every spelling, and only that', () => { + expect( + redactSecretShapedValue({ + args: ['-u', 'alice:Qz7pw', '-ualice:Qz7pw', '--user=bob:Qz7pw', '--user', 'carol:Qz7pw'], + }), + ).toEqual({ + args: ['-u', '[redacted]', '-u[redacted]', '--user=[redacted]', '--user', '[redacted]'], + }); + const benign = { + args: [ + ...['-u', 'alice', '--user', 'carol', '-u', '1000:1000', '--user=dave', '-uroot'], + ...['--user', 'git+https://example.invalid/r.git'], + ], + }; + expect(redactSecretShapedValue(benign)).toEqual(benign); + // A value that is not a user:password is still read by every other rule. + const key = `sk-${'Q'.repeat(20)}`; + expect( + JSON.stringify(redactSecretShapedValue({ args: ['--user', key, '-u', key] })), + ).not.toContain(key); + }); + it('leaves ordinary structured args alone — the negative control', () => { // Without this the rule above is satisfied by redacting everything, which would strip the stored row of // the diagnostic value that is its whole remaining purpose. @@ -1242,12 +1336,12 @@ describe('PR95 raw ?/# userinfo in the journal projection', () => { // Core retains a colonless userinfo under 16 characters; the LLM diagnostic backstop masks it. expect(redactSecretShapedText('http://#TOKEN@h.example')).toBe('http://#TOKEN@h.example'); expect(scrubSecrets('http://#TOKEN@h.example')).toBe('http://[REDACTED]@h.example'); - // Core's authority also ends at `"`, `'`, `,`, `}` and `]`, so it leaves userinfo containing - // one of them as it is: an already-masked password, a bracketed-host prefix, a raw delimiter. + // Core's authority also ends at `"`, `'`, `}` and `]`, the delimiters of the JSON and quoted texts it + // reads, so it leaves userinfo containing one of them as it is: an already-masked password, a + // bracketed-host prefix, a raw delimiter. for (const [input, masked] of [ ['https://app:[REDACTED]@h.example/', 'https://[REDACTED]@h.example/'], ['https://[::1]:8443?u:TOKENTOKEN@h.example/', 'https://[REDACTED]@h.example/'], - ['https://user:pa,ssword@host.example/', 'https://[REDACTED]@host.example/'], ['https://user:pa"ssword@host.example/', 'https://[REDACTED]@host.example/'], ["https://user:pa'ssword@host.example/", 'https://[REDACTED]@host.example/'], ['https://user:pa}ssword@host.example/', 'https://[REDACTED]@host.example/'], @@ -1257,6 +1351,37 @@ describe('PR95 raw ?/# userinfo in the journal projection', () => { } }); + it('reads a `,` or `/` in a password through its `@`, never past a port, query or other delimiter', () => { + // A PR #96 review found the authority scan leaving `ssword` and `ss` public here. + for (const [input, expected] of [ + ['https://user:pa,ssword@host.example/', 'https://[redacted]@host.example/'], + ['https://user:pa/ss@host.example/x', 'https://[redacted]@host.example/x'], + ['{"u":"https://user:p,w/x@h.example/v1"}', '{"u":"https://[redacted]@h.example/v1"}'], + ['https://user:pa/ss@host.example/@scope/pkg', 'https://[redacted]@host.example/@scope/pkg'], + // A raw `?` that is not a host's public query stays userinfo, as the authority scan reads it. + ['https://u:p?x/y@h.example/', 'https://[redacted]@h.example/'], + ] as const) { + expect(redactSecretShapedText(input), input).toBe(expected); + } + // The backstop already reads a comma; both now mask it. + expect(scrubSecrets('https://user:pa,ssword@host.example/')).toBe( + 'https://[REDACTED]@host.example/', + ); + for (const input of [ + 'https://host.example:8080/a@b.example', + 'https://host.example:8080,bob@x.example', + 'https://host.example:8080?next=/a@b.example', + 'https://example.com/@user', + 'https://user:pa,ss/x', + '{"u":"https://h.example:p","e":"x@y.example"}', + "'https://u:pw/x', 'a@b.example'", + 'https://a:b/https://c@d.example', + `https://user:pa/${'s'.repeat(600)}@host.example/`, + ]) { + expect(redactSecretShapedText(input), input).toBe(input); + } + }); + it('agrees with the LLM backstop on every raw ?/# decision outside those differences', () => { for (const input of [ 'https://alice:pa?ss@provider.invalid/v1', diff --git a/packages/core/src/tools/bounding.ts b/packages/core/src/tools/bounding.ts index 4adf83a0..9d86580a 100644 --- a/packages/core/src/tools/bounding.ts +++ b/packages/core/src/tools/bounding.ts @@ -55,8 +55,9 @@ function countLines(text: string): number { * I3 gap where a `read_media` (or any) tool result carrying a `{ kind:'base64', data }` source would be * JSON-stringified into the `agent:tool_result.outputSummary` event / the spill / the over-cap preview — * a durable/event boundary the emit-time `deInlineMedia` choke point cannot catch, since it sees only the - * resulting flat string, not a structured media source). The MODEL-facing result value is left intact - * (the model still receives the bytes via the seam); only this text projection is redacted. Cycle-safe. + * resulting flat string, not a structured media source). Only this text projection is redacted; the + * model-facing value keeps the bytes while it fits the ceiling, and `boundForModel` counts the omitted bytes + * against that ceiling ({@link OmittedMedia}). Cycle-safe. */ /** A PLAIN object (Object/null prototype) — NOT a Date/RegExp/Map/Set/class instance, which JSON.stringify * renders natively and the walk must leave untouched (else a Date would collapse to `{}`). Narrows without an `as`. */ @@ -78,9 +79,25 @@ export function redactInlineMedia(value: unknown): unknown { return redactInlineMediaForText(value, new WeakSet()); } -function redactInlineMediaForText(value: unknown, seen: WeakSet): unknown { +/** + * The size of the inline media a text projection replaced with a marker. The model-facing value still holds + * those bytes — a builtin's media rides `attachMedia` instead (ADR-0089 §1), so media inside a dispatch + * result is ordinary text to the model and to the effect journal. A PR #96 review found a 30 MiB data URI + * measured as its 25-character marker, passed whole to the model and settled into `history.db`. + */ +interface OmittedMedia { + bytes: number; +} + +function redactInlineMediaForText( + value: unknown, + seen: WeakSet, + omitted?: OmittedMedia, +): unknown { if (typeof value === 'string') { - return isBase64DataUri(value) ? '[base64 data URI omitted]' : value; + if (!isBase64DataUri(value)) return value; + if (omitted !== undefined) omitted.bytes += utf8ByteLength(value); + return '[base64 data URI omitted]'; } if (typeof value !== 'object' || value === null) { return value; @@ -91,9 +108,10 @@ function redactInlineMediaForText(value: unknown, seen: WeakSet): unknow seen.add(value); try { if (Array.isArray(value)) { - return value.map((item) => redactInlineMediaForText(item, seen)); + return value.map((item) => redactInlineMediaForText(item, seen, omitted)); } if (isBinaryBuffer(value)) { + if (omitted !== undefined) omitted.bytes += binaryByteLength(value); return '[binary buffer omitted]'; // a raw Uint8Array/ArrayBuffer — never JSON the decimal byte values } if (!isPlainObject(value)) { @@ -101,14 +119,16 @@ function redactInlineMediaForText(value: unknown, seen: WeakSet): unknow } if (isCanonicalBase64Source(value)) { const data = value['data']; + const base64Length = typeof data === 'string' ? data.length : 0; + if (omitted !== undefined) omitted.bytes += base64Length; // Report the base64 CHARACTER length (≈1.33× the byte count), not "bytes" — observability only. - return { kind: 'base64', base64Length: typeof data === 'string' ? data.length : 0 }; + return { kind: 'base64', base64Length }; } const out: Record = {}; for (const [key, item] of Object.entries(value)) { // Preserve an own data key such as __proto__; assignment would invoke an inherited setter. Object.defineProperty(out, key, { - value: redactInlineMediaForText(item, seen), + value: redactInlineMediaForText(item, seen, omitted), enumerable: true, configurable: true, writable: true, @@ -121,6 +141,12 @@ function redactInlineMediaForText(value: unknown, seen: WeakSet): unknow } } +/** A binary buffer's byte length; every kind `isBinaryBuffer` admits has an own `byteLength`. */ +function binaryByteLength(value: object): number { + const length: unknown = Reflect.get(value, 'byteLength'); + return typeof length === 'number' ? length : 0; +} + // One definition builds both the complete matcher and its necessary-prefix preflight. Prefixes // have bounded width: testing their absence must never run the unbounded JWT/body alternatives. // One alternation still preserves outer spans when credential families overlap. Bounded lookaheads @@ -683,6 +709,13 @@ function visitEscapedSeparators(text: string, report: SecretSpanVisitor): void { visitSchemeCredentials(view, report); } +/** The separator each escape letter stands for. */ +const ESCAPED_SEPARATORS: ReadonlyMap = new Map([ + ['n', '\n'], + ['r', '\r'], + ['t', '\t'], +]); + /** * `text` with each escaped line feed, carriage return or tab (`\n`, `\r`, `\t`) replaced by that separator * twice, so offsets stay valid. The backslash before the letter may end a longer run, as in a JSON text @@ -692,17 +725,16 @@ function visitEscapedSeparators(text: string, report: SecretSpanVisitor): void { function escapedSeparatorView(text: string): string { const parts: string[] = []; let copied = 0; - for (let index = text.indexOf('\\'); index >= 0; index = text.indexOf('\\', index)) { + let index = text.indexOf('\\'); + while (index >= 0) { let end = index; while (text[end] === '\\') end += 1; - const letter = text[end]; - const separator = - letter === 'n' ? '\n' : letter === 'r' ? '\r' : letter === 't' ? '\t' : undefined; + const separator = ESCAPED_SEPARATORS.get(text[end] ?? ''); if (separator !== undefined) { parts.push(text.slice(copied, end - 1), separator, separator); copied = end + 1; } - index = end; + index = text.indexOf('\\', end); } if (parts.length === 0) return text; parts.push(text.slice(copied)); @@ -904,9 +936,17 @@ function visitSecretKeyValues( const readings = kept === undefined ? [quotedValueEnd(values, valueStart), ...plain] : plain; // A scheme label's token can take the line break before a closer that a reading reaches early; a // value that no reading closes is decided once every range of the pass is known instead. - const first = readings.reduce(laterClosing); + // `readings` holds the value's own reading first; the fallback states that here. + const first = readings + .slice(1) + .reduce( + (later, reading) => laterClosing(later, reading), + readings[0] ?? quotedValueEnd(opener, valueStart), + ); const read = first.closed - ? views.schemed.map((view) => quotedValueEnd(view, valueStart)).reduce(laterClosing, first) + ? views.schemed + .map((view) => quotedValueEnd(view, valueStart)) + .reduce((later, reading) => laterClosing(later, reading), first) : first; const kind = escaped ? `\\${opening}` : opening; if (!throughCloser) { @@ -1100,11 +1140,12 @@ const HOST_SHAPED_MAX_LENGTH = 260; * its last `@`. This projection masks the userinfo and reads every key and parameter in it but its * username's; their values keep their own extent and the rules' ranges merge * (`https://gateway?password=P@ss.word1` projects to `https://[redacted]`). - * The projection differs from that backstop by design: its authority also ends at `"`, `'`, `,`, - * `}` and `]`, so userinfo containing one of them (an already-masked `[REDACTED]` password among - * them) is left as it is; it recognizes any scheme of up to 32 characters rather than a finite list; - * and a userinfo with no password is a vanity handle below 16 characters and masked from 16 on, - * under every scheme. + * The projection differs from that backstop by design: its authority also ends at `"`, `'`, + * `}` and `]`, the delimiters of the JSON and quoted texts it reads, so userinfo containing one of + * them (an already-masked `[REDACTED]` password among them) is left as it is; it recognizes any + * scheme of up to 32 characters rather than a finite list; and a userinfo with no password is a + * vanity handle below 16 characters and masked from 16 on, under every scheme. A `,` or `/` in a + * password is read on to its `@` ({@link passwordPastDelimiter}). */ function isPublicQueryContact(userinfo: string, query: number): boolean { // A longer prefix cannot be a hostname, and an unbounded dotted run would exhaust V8's RegExp @@ -1113,6 +1154,31 @@ function isPublicQueryContact(userinfo: string, query: number): boolean { return HOST_SHAPED_AUTHORITY.test(userinfo.slice(0, query)) && userinfo.includes('=', query + 1); } +/** + * The `@` of a password that holds a `,` or `/`, or -1. The authority scan stops at either, so + * `https://user:pa,ss@host` and `https://user:pa/ss@host` left the password's tail public (a PR #96 + * review finding; RFC 3986 admits a raw `,` in userinfo, and a raw `/` is a malformed URL a + * credential can still sit in). Reads on past them only, to the first `@` within 512 characters. A + * port (`host:8080/a@b`, `host:8080,bob@x.io`), a public query contact as the authority scan reads + * one ({@link isPublicQueryContact}), another scheme, and the delimiters the authority keeps + * (whitespace, quotes, `}`, `]`) keep the text public. + */ +function passwordPastDelimiter(text: string, start: number, end: number): number { + if (text[end] !== ',' && text[end] !== '/') return -1; + const authority = text.slice(start, end); + const colon = authority.indexOf(':'); + if (colon < 0 || /^\d*$/.test(authority.slice(colon + 1))) return -1; + const query = authority.search(/[?#]/); + if (query >= 0 && isPublicQueryContact(authority, query)) return -1; + PASSWORD_TAIL.lastIndex = end; + const tail = PASSWORD_TAIL.exec(text)?.[0]; + if (tail === undefined || tail.includes('://')) return -1; + return end + tail.length - 1; +} + +/** A password's tail past a `,` or `/`, through its `@`; bounded so a long run is never rescanned. */ +const PASSWORD_TAIL = /[^\s"'}\]?#@]{0,512}@/y; + function visitUrlUserinfo(text: string, mask: SecretSpanVisitor): void { if (!text.includes('://')) return; const scheme = /[a-z][\w+.-]{0,31}:\/\//gi; @@ -1125,7 +1191,16 @@ function visitUrlUserinfo(text: string, mask: SecretSpanVisitor): void { const firstQuery = authority.search(/[?#]/); const authorityAt = firstQuery < 0 ? -1 : authority.lastIndexOf('@', firstQuery - 1); const at = authorityAt >= 0 ? authorityAt : authority.lastIndexOf('@'); - if (at < 0) continue; + if (at < 0) { + const passwordAt = passwordPastDelimiter(text, start, end); + if (passwordAt >= 0) { + // Its username owns the text before the first ':', as below. + const url = { schemeStart: match.index, ownsUntil: start + authority.indexOf(':') }; + mask(start, passwordAt, '[redacted]', url); + scheme.lastIndex = passwordAt + 1; + } + continue; + } const userinfo = authority.slice(0, at); const colon = userinfo.indexOf(':'); const query = userinfo.search(/[?#]/); @@ -1306,12 +1381,18 @@ function isSecretishKey(key: string): boolean { * answers one with the other's recorded result — the exact failure the digest exists to prevent. */ function keyWords(key: string): readonly string[] { - return key + return nameWords(key).map((word) => + word.endsWith('s') && word.length > 3 ? word.slice(0, -1) : word, + ); +} + +/** A name's lowercase words as written, before {@link keyWords} folds plurals. */ +function nameWords(name: string): readonly string[] { + return name .replace(/([a-z0-9])([A-Z])/g, '$1 $2') .split(/[^A-Za-z0-9]+/) .filter((word) => word.length > 0) - .map((word) => word.toLowerCase()) - .map((word) => (word.endsWith('s') && word.length > 3 ? word.slice(0, -1) : word)); + .map((word) => word.toLowerCase()); } /** Names that mean "this is a credential" on their own. */ @@ -1361,35 +1442,59 @@ function redactArgVector( let redactNext: string | undefined; for (const item of items) { if (redactNext !== undefined) { - out.push(redactCredentialArgument(redactNext, item, whole)); + const masked = redactCredentialArgument(redactNext, item, whole); + out.push(masked ?? redactSecretShapedWalk(item, seen, project, collide, whole)); redactNext = undefined; continue; } if (typeof item === 'string') { const separator = item.indexOf('='); const flag = separator > 0 ? item.slice(0, separator) : ''; - if (isSecretishFlag(flag)) { - const value = item.slice(separator + 1); - out.push(`${flag}=${redactCredentialArgument(flag, value, whole)}`); + if (isCredentialFlag(flag)) { + const masked = redactCredentialArgument(flag, item.slice(separator + 1), whole); + if (masked !== undefined) { + out.push(`${flag}=${masked}`); + continue; + } + } + const glued = gluedUserPassword(item); + if (glued !== undefined) { + out.push(`-u${whole(glued, '[redacted]')}`); continue; } - if (isSecretishFlag(item)) redactNext = item; + if (isCredentialFlag(item)) redactNext = item; } out.push(redactSecretShapedWalk(item, seen, project, collide, whole)); } return out; } -/** One credential-value rule for both argv spellings; only ambiguous -p may name a public port. */ -function redactCredentialArgument(flag: string, value: unknown, whole: WholeMask): string { +/** + * One credential-value rule for both argv spellings; only ambiguous -p may name a public port. `undefined` + * means the value is not a credential after all (a `-u`/`--user` name without a password) and is walked as + * any other element. + */ +function redactCredentialArgument( + flag: string, + value: unknown, + whole: WholeMask, +): string | undefined { if (flag.toLowerCase() === '-p' && typeof value === 'string' && looksLikePort(value)) { return value; } + if (isUserPasswordFlag(flag)) { + return looksLikeUserPassword(value) ? whole(value, '[redacted]') : undefined; + } return whole(value, '[redacted]'); } +/** A flag whose next argument (or `=` value) is a credential, or a `user:password` pair. */ +function isCredentialFlag(flag: string): boolean { + return isSecretishFlag(flag) || isUserPasswordFlag(flag); +} + /** - * A CLI flag that introduces a credential — `--token`, `--password`, `-p`. + * A CLI flag that introduces a credential — `--token`, `--password`, `--auth-token`, `-p`. * * `-p` is deliberately kept AND deliberately qualified. It really is the password flag for `mysql` and * `psql`, and dropping it would leak those verbatim — but it is also the port flag for `ssh` and the @@ -1398,7 +1503,27 @@ function redactCredentialArgument(flag: string, value: unknown, whole: WholeMask * `-p` ambiguity retains that exception; explicitly named credential flags do not. Both separate and * `--password=…` argument spellings use the entire value, rather than a partial text-pattern match. * See {@link looksLikePort}. + * + * Besides the anchored names, a flag counts when its LAST word names a credential, read with the object-key + * rule's words ({@link SECRET_WORDS}, or `key` qualified by the word before it): `--auth-token`, + * `--access-token`, `--db-password`, `--pwd`, `--x-api-key`. A PR #96 review found those passing verbatim + * while the key rule masked the same names as object keys. The last word, not any word, because a flag + * that only MENTIONS a credential is usually a boolean or names a file — `--password-stdin`, `--token-file` + * — and masking the element after it would make calls that differ only there share one journal identity. + * For the same reason the word is read as written, without the key rule's plural folding (`--max-tokens` + * is a count), with `credentials` the one plural admitted, and a `no-` flag (`--no-password`) never counts. */ +function isSecretishFlag(item: string): boolean { + if (SECRETISH_FLAG.test(item)) return true; + const name = /^--?([A-Za-z][\w-]{0,63})$/.exec(item)?.[1]; + if (name === undefined) return false; + const words = nameWords(name); + if (words[0] === 'no') return false; + const last = words.at(-1) ?? ''; + if (SECRET_WORDS.has(last) || last === 'credentials') return true; + return last === 'key' && KEY_QUALIFIERS.has(words.at(-2) ?? ''); +} + /** * The flag names, as one anchored alternation. * @@ -1409,8 +1534,31 @@ function redactCredentialArgument(flag: string, value: unknown, whole: WholeMask const SECRETISH_FLAG = /^-{1,2}(?:p|pw|password|passwd|secret|token|api[_-]?key|auth|access[_-]?key|private[_-]?key|client[_-]?secret|credential|passphrase|pin)$/i; // NOSONAR — the metric is counting NAMES; see above -function isSecretishFlag(item: string): boolean { - return SECRETISH_FLAG.test(item); +/** + * curl's `-u`/`--user` carries `user:password` in ONE argument, so its value is a credential only when it + * holds a `:`. A `uid[:gid]` (`docker run -u 1000:1000`) and a URL (`pip install --user git+https://…`) + * are public. + */ +function isUserPasswordFlag(flag: string): boolean { + return flag === '-u' || flag === '--user'; +} + +function looksLikeUserPassword(value: unknown): value is string { + return ( + typeof value === 'string' && + value.includes(':') && + !value.includes('://') && + !/^\d+(?::\d+)?$/.test(value) + ); +} + +/** The value of curl's glued `-ualice:secret` spelling, when it is a `user:password` pair. */ +function gluedUserPassword(item: string): string | undefined { + if (!item.startsWith('-u') || item.length < 3 || item[2] === '-' || item[2] === '=') { + return undefined; + } + const value = item.slice(2); + return looksLikeUserPassword(value) ? value : undefined; } /** Port-shaped values disambiguate `-p`; an explicit credential flag has no port exception. */ @@ -1500,13 +1648,19 @@ function redactSecretShapedWalk( * the context it is read in; projecting each string alone would mask a key or a PEM marker that a later * member's text needs. */ -function toText(result: unknown): { readonly text: string; readonly summaryText: string } { +function toText(result: unknown): { + readonly text: string; + readonly summaryText: string; + /** The inline media bytes the texts replaced with markers but the model-facing `result` still holds. */ + readonly omittedBytes: number; +} { + const omitted: OmittedMedia = { bytes: 0 }; if (typeof result === 'string') { const text = isBase64DataUri(result) ? '[base64 data URI omitted]' : result; - return { text, summaryText: text }; + return { text, summaryText: text, omittedBytes: text === result ? 0 : utf8ByteLength(result) }; } if (result === undefined) { - return { text: '', summaryText: '' }; + return { text: '', summaryText: '', omittedBytes: 0 }; } let safe: unknown; try { @@ -1515,11 +1669,15 @@ function toText(result: unknown): { readonly text: string; readonly summaryText: // URIs, AND raw binary buffers at any nesting before JSON-serialisation, so no media bytes (base64 or // decimal) can ride the summary/spill/preview (I3). A non-media object is returned as a structural clone // (same JSON). Both texts are made from this one walk. - safe = redactInlineMediaForText(result, new WeakSet()); + safe = redactInlineMediaForText(result, new WeakSet(), omitted); } catch { - return { text: '[unserializable]', summaryText: '[unserializable]' }; // throwing getter during redaction + return { text: '[unserializable]', summaryText: '[unserializable]', omittedBytes: 0 }; // throwing getter during redaction } - return { text: jsonText(() => safe), summaryText: structuredSummaryText(safe) }; + return { + text: jsonText(() => safe), + summaryText: structuredSummaryText(safe), + omittedBytes: omitted.bytes, + }; } /** @@ -1667,9 +1825,11 @@ export async function boundForModel( host: ToolHost, signal?: AbortSignalLike, ): Promise { - const { text, summaryText } = toText(result); + const { text, summaryText, omittedBytes } = toText(result); const summary = makeSummary(summaryText); - const bytes = utf8ByteLength(text); + // The ceiling is the model-facing value's, so inline media the texts omit still counts; the spill and the + // preview below are made from the media-free text, so an over-ceiling result never hands them base64. + const bytes = utf8ByteLength(text) + omittedBytes; const lines = countLines(text); if (bytes <= limits.maxBytes && lines <= limits.maxLines) { @@ -1696,5 +1856,10 @@ export async function boundForModel( ref === undefined ? `\n\n[… truncated: ${bytes} bytes / ${lines} lines; full output unavailable (${unavailableNote}) …]` : `\n\n[… truncated: ${bytes} bytes / ${lines} lines; full output at ${ref} …]`; - return { value: makePreview(text, limits) + marker, truncated: true, summary }; + // Over the ceiling only by the media it omits, the text itself is the preview, not a doubled head and tail. + const preview = + utf8ByteLength(text) <= limits.maxBytes && lines <= limits.maxLines + ? text + : makePreview(text, limits); + return { value: preview + marker, truncated: true, summary }; } diff --git a/packages/core/src/tools/registry.test.ts b/packages/core/src/tools/registry.test.ts index 4fa3f304..0a424e7d 100644 --- a/packages/core/src/tools/registry.test.ts +++ b/packages/core/src/tools/registry.test.ts @@ -439,6 +439,29 @@ describe('ToolRegistry — git_commit human-gate', () => { expect(prepare).toHaveBeenCalledTimes(1); expect(settle).toHaveBeenCalledTimes(1); }); + + it('settles a bounded preview, not a 30 MiB data URI, into the effect journal', async () => { + // A PR #96 review: the ceiling was measured on the data URI's text marker, so the whole value reached + // the model and `settleCommitted` wrote it to `history.db`. + const uri = `data:image/png;base64,${'A'.repeat(30 * 1024 * 1024)}`; + const spawn = vi.fn['spawn']>(() => + Promise.resolve({ exitCode: 0, stdout: uri, stderr: '', durationMs: 1 }), + ); + const context = ctx({ toolPolicy: { allowedCommands: ['ls'] } }); + const settle = vi.fn(context.effects.settle); + const out = await registry(stubHost({ process: { spawn } })).dispatch( + call('run_command', { command: 'ls' }), + { + ...context, + effects: { ...context.effects, settle }, + }, + ); + expect(out.events.result.success).toBe(true); + expect(settle).toHaveBeenCalledTimes(1); + const settled = JSON.stringify(settle.mock.calls[0]?.[3]); + expect(settled).not.toContain('AAAAAAAA'); + expect(settled.length).toBeLessThan(64 * 1024); + }); }); /* --- per-tool approval (ADR-0057 EA3) --- */ diff --git a/tools/causal-controls/provenance.test.mjs b/tools/causal-controls/provenance.test.mjs index 3ef0613b..f07879d7 100644 --- a/tools/causal-controls/provenance.test.mjs +++ b/tools/causal-controls/provenance.test.mjs @@ -358,8 +358,8 @@ test('secrets workload corrections retain independent complete-count and reader- // Sixth pre-review and round 9: a summary's escaped separators and its unwalked reading. 'core-summary-escape-view', 'core-summary-pem-markers', - ].map((id) => [id, ['packages/core/src/tools/bounding.test.ts'], 221]), - ['core-unsettled-refusal', ['packages/core/src/tools/registry.test.ts'], 115], + ].map((id) => [id, ['packages/core/src/tools/bounding.test.ts'], 234]), + ['core-unsettled-refusal', ['packages/core/src/tools/registry.test.ts'], 116], ...[ 'diagnostic-joined-before-masking', 'diagnostic-joined-original-boundary', diff --git a/tools/secrets-controls/README.md b/tools/secrets-controls/README.md index 6cdd960a..14f8440a 100644 --- a/tools/secrets-controls/README.md +++ b/tools/secrets-controls/README.md @@ -178,8 +178,8 @@ modern `sk-` body characters, joined-span application, the original-text left-bo check, split scheme labels, split API-header labels and raw `?`/`#` password recognition independently. Four LLM controls remove the before-masking LF/TAB probe, its original left-boundary check, split API-header recognition and raw-password recognition. -Every core projection phase executes the complete 221-case bounding file (the journal-identity -control's phases the complete 115-case registry file); every new LLM phase executes the complete 224-case scrubber file. Independent ordinary tooling tests pin +Every core projection phase executes the complete 234-case bounding file (the journal-identity +control's phases the complete 116-case registry file); every new LLM phase executes the complete 224-case scrubber file. Independent ordinary tooling tests pin these inventories, the 278-case overlap cohort, 180-case CLI diagnostic cohort and 18-case command-export cohort. Actual collection still has to match in every phase. diff --git a/tools/secrets-controls/mutations.mjs b/tools/secrets-controls/mutations.mjs index d164c289..c7008a2f 100644 --- a/tools/secrets-controls/mutations.mjs +++ b/tools/secrets-controls/mutations.mjs @@ -102,20 +102,20 @@ const schemeLabel = String.raw`B[\n\t]*e[\n\t]*a[\n\t]*r[\n\t]*e[\n\t]*r|B[\n\t] const headerName = String.raw`(?:(?:x-)?api-key|x-goog-api-key)\\*["']?\s*:`; // The header rule, piece by piece, so a control can remove or weaken exactly one guard. const header = { - boundary: String.raw`(?[^\s"',;}\\][^\s"',;}]*?)`, stop: String.raw`(?=(?<=[^A-Za-z0-9_-])${headerName}|\\{0,15}[\s"',;}]|\\{0,15}$)`, }; const headerLine = (part = {}) => { const value = { ...header, ...part }; return ( - String.raw` /${value.boundary}(?:(?:x-)?api-key|x-goog-api-key)${value.separator}${value.labelRun}` + - String.raw`${value.notLabel}${value.prefix}${value.notNested}${value.token}${value.stop}/dgi,` + ` /${value.boundary}(?:(?:x-)?api-key|x-goog-api-key)${value.separator}${value.labelRun}` + + `${value.notLabel}${value.prefix}${value.notNested}${value.token}${value.stop}/dgi,` ); }; // The one standalone label rule, piece by piece; its first label is either label. @@ -132,8 +132,8 @@ const label = { const labelLine = (part = {}) => { const value = { ...label, ...part }; return ( - String.raw` /${value.first}${value.concat}${value.close}${value.notLabel}${value.lead}${value.token}` + - String.raw`${value.trail}${value.stop}/dgi,` + ` /${value.first}${value.concat}${value.close}${value.notLabel}${value.lead}${value.token}` + + `${value.trail}${value.stop}/dgi,` ); }; const mergedMasking = [ @@ -143,7 +143,8 @@ const mergedMasking = [ ' );', ].join('\n'); const queryRule = (names) => String.raw`/[?&#](?:${names})=(?[^&\s#]+)/dgi`; -const queryNames = String.raw`(?:x[-_])?api[-_]?key|api[-_]?secret|(?:access|secret)[-_]?key|(?:access|refresh|id|auth|session)[-_]?token|client[-_]?secret|key|token|auth|password|secret`; +const queryNames = + '(?:x[-_])?api[-_]?key|api[-_]?secret|(?:access|secret)[-_]?key|(?:access|refresh|id|auth|session)[-_]?token|client[-_]?secret|key|token|auth|password|secret'; const overlapMerge = ' if (open !== undefined && range.start < open.end) {'; const mergeExtent = ' open.end = Math.max(open.end, range.end);'; const hostFollowsLine = @@ -174,7 +175,8 @@ const contactAssignmentCases = (secondInput) => ); const contactHostCase = 'masks userinfo whose raw ?/# follows no host-shaped prefix: https://us?er=1:TOKEN@h.example/'; -const contactAssignment = String.raw` return HOST_SHAPED_AUTHORITY.test(userinfo.slice(0, query)) && userinfo.includes('=', query + 1);`; +const contactAssignment = + " return HOST_SHAPED_AUTHORITY.test(userinfo.slice(0, query)) && userinfo.includes('=', query + 1);"; const connectionSchemeMarker = 'SECRETS_B2_CONNECTION_SCHEME'; // sftp also matches the original list by its ftp suffix, so the old list still masks it. const connectionSchemeCases = connectionSchemes @@ -334,7 +336,7 @@ const withoutCoreOriginalBoundary = coreOriginalBoundary ) .replace(" if (boundary?.test(text[originalStart - 1] ?? '')) return;", ''); /** The complete core projection test file every core projection control runs. */ -const CORE_PROJECTION_CASES = 221; +const CORE_PROJECTION_CASES = 234; function coreProjectionControl(id, before, after, targets, marker = coreProjectionMarker) { return control( id, @@ -349,7 +351,7 @@ function coreProjectionControl(id, before, after, targets, marker = coreProjecti ); } /** The complete registry test file the journal-identity control runs. */ -const REGISTRY_CASES = 115; +const REGISTRY_CASES = 116; /** The complete scrubber test file every LLM diagnostic control runs. */ const LLM_SCRUB_CASES = 224; function diagnosticControl(id, before, after, targets, marker = diagnosticMarker) { @@ -468,7 +470,7 @@ export const mutations = [ 'query-names', scrubSecretsLeaf, queryRule(queryNames), - queryRule(String.raw`(?:x[-_])?api[-_]?key|key|token|access[-_]?token|auth|password|secret`), + queryRule('(?:x[-_])?api[-_]?key|key|token|access[-_]?token|auth|password|secret'), ['packages/llm/src/llm-error.test.ts'], 98, 'SECRETS_CAUSAL_QUERY_NAMES masks each added family without another credential shape', @@ -1226,7 +1228,7 @@ export const mutations = [ ['diagnostic-header-not-nested', { notNested: '' }], [ 'diagnostic-header-nested-stop', - { stop: header.stop.replace(String.raw`(?<=[^A-Za-z0-9_-])${headerName}|`, '') }, + { stop: header.stop.replace(`(?<=[^A-Za-z0-9_-])${headerName}|`, '') }, ], ['diagnostic-header-lookbehind', { boundary: '(?:^|[^A-Za-z0-9_-])' }], ].map(([id, part]) => @@ -1405,7 +1407,7 @@ export const mutations = [ diagnosticControl( 'diagnostic-userinfo-contact-assignment', contactAssignment, - String.raw` return HOST_SHAPED_AUTHORITY.test(userinfo.slice(0, query));`, + ' return HOST_SHAPED_AUTHORITY.test(userinfo.slice(0, query));', contactAssignmentCases('https://host.example#sec@x.example'), userinfoContactMarker, ), @@ -1413,7 +1415,7 @@ export const mutations = [ 'diagnostic-userinfo-contact-host', scrubSecretsLeaf, contactAssignment, - String.raw` return userinfo.includes('=', query + 1);`, + " return userinfo.includes('=', query + 1);", ['packages/llm/src/scrub-secrets.test.ts'], LLM_SCRUB_CASES, contactHostCase, @@ -1423,7 +1425,7 @@ export const mutations = [ coreProjectionControl( 'core-userinfo-contact-assignment', contactAssignment, - String.raw` return HOST_SHAPED_AUTHORITY.test(userinfo.slice(0, query));`, + ' return HOST_SHAPED_AUTHORITY.test(userinfo.slice(0, query));', contactAssignmentCases('https://provider.example#section@x.example'), coreUserinfoContactMarker, ), @@ -1431,7 +1433,7 @@ export const mutations = [ 'core-userinfo-contact-host', coreProjection, contactAssignment, - String.raw` return userinfo.includes('=', query + 1);`, + " return userinfo.includes('=', query + 1);", coreProjectionCases, CORE_PROJECTION_CASES, contactHostCase, @@ -1693,7 +1695,7 @@ export const mutations = [ ), diagnosticControl( 'diagnostic-url-split-retry', - " /[\\n\\t]/.test(text[originalOffset + 1] ?? '') &&", + String.raw` /[\n\t]/.test(text[originalOffset + 1] ?? '') &&`, ' false &&', ['blanks an LF/TAB-split ftp credential after a line that ends in s'], 'SECRETS_B5_SPLIT_SFTP', From b4ef9374e609f3a4a6ff6d71d04b21d7342be7bc Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Wed, 4 Nov 2026 13:56:27 +0300 Subject: [PATCH 90/97] fix(cli): mask secret shapes in the run failure summary The final summary's error row collapsed separators with sanitizeInline but did not mask, unlike every other sink for untrusted diagnostics. It now uses sanitizeUntrustedInline, with a regression test (PR #96 review, low). Co-Authored-By: Claude Opus 5.5 --- apps/cli/src/render/tui/final-summary.test.ts | 18 ++++++++++++++++++ apps/cli/src/render/tui/final-summary.ts | 5 +++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/apps/cli/src/render/tui/final-summary.test.ts b/apps/cli/src/render/tui/final-summary.test.ts index 7476649d..8bda7e32 100644 --- a/apps/cli/src/render/tui/final-summary.test.ts +++ b/apps/cli/src/render/tui/final-summary.test.ts @@ -221,4 +221,22 @@ describe('renderFinalSummary', () => { expect(lines[1]).toContain('✓ deploy completed'); expect(lines).not.toContain(' ✓ deploy completed'); }); + + it('masks a secret shape in the failure message before it reaches the terminal', () => { + // A PR #96 review: the summary row collapsed separators but did not mask, unlike every other + // untrusted diagnostic sink. + const key = `sk-${'Qz7'.repeat(8)}`; + const state: RunViewState = { + ...initialRunViewState(), + summary: { + outcome: 'failed', + errorMessage: `provider rejected Authorization: Bearer ${key}`, + }, + }; + + const output = renderFinalSummary(state); + expect(output).not.toContain(key); + expect(output).toContain('provider rejected'); + expect(output).toContain('[REDACTED]'); + }); }); diff --git a/apps/cli/src/render/tui/final-summary.ts b/apps/cli/src/render/tui/final-summary.ts index 535bf44f..7d7e820c 100644 --- a/apps/cli/src/render/tui/final-summary.ts +++ b/apps/cli/src/render/tui/final-summary.ts @@ -7,7 +7,7 @@ import { } from './format.js'; import { nodeSuffix } from './projection.js'; import type { RunViewState } from './run-view-model.js'; -import { sanitizeInline } from '../sanitize.js'; +import { sanitizeInline, sanitizeUntrustedInline } from '../sanitize.js'; /** * Render the **persistent** final summary the `ink` renderer writes after it unmounts (workstream **2.E**). @@ -57,7 +57,8 @@ export function renderFinalSummary(state: RunViewState): string { if (summary?.errorMessage !== undefined) { // A summary error occupies ONE structural row. Collapse line/column separators as well as terminal // controls, so provider/model text cannot forge a believable node-status row beneath the headline. - lines.push(` ${sanitizeInline(summary.errorMessage)}`); + // It is untrusted text (a provider or tool diagnostic), so its secret shapes are masked first. + lines.push(` ${sanitizeUntrustedInline(summary.errorMessage)}`); } for (const id of state.nodeOrder) { From 8445a7007309993b8bed4dadfb306e1b42bd5c61 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Wed, 4 Nov 2026 13:56:27 +0300 Subject: [PATCH 91/97] chore(cli): typecheck the build configuration tsup.config.ts decides whether a build carries source maps, yet no tsc program covered it (PR #96 review, low). A second typecheck program now does, and the one index-signature access it found uses bracket notation. Co-Authored-By: Claude Opus 5.5 --- apps/cli/package.json | 2 +- apps/cli/tsconfig.config.json | 10 ++++++++++ apps/cli/tsup.config.ts | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 apps/cli/tsconfig.config.json diff --git a/apps/cli/package.json b/apps/cli/package.json index e6a9cfce..f6842501 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -40,7 +40,7 @@ "scripts": { "build": "tsup", "prepack": "node ../../tools/cli-package/prepack.mjs", - "typecheck": "tsc -p tsconfig.json --noEmit", + "typecheck": "tsc -p tsconfig.json --noEmit && tsc -p tsconfig.config.json --noEmit", "lint": "eslint src", "test": "vitest run" }, diff --git a/apps/cli/tsconfig.config.json b/apps/cli/tsconfig.config.json new file mode 100644 index 00000000..dc1c1fba --- /dev/null +++ b/apps/cli/tsconfig.config.json @@ -0,0 +1,10 @@ +{ + // The build configuration is not under `src/`, so `tsconfig.json` does not check it, yet `tsup.config.ts` + // decides whether a build carries source maps (a candidate build) or is map-free. Typecheck it too. + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "noEmit": true, + "types": ["node"] + }, + "include": ["tsup.config.ts"] +} diff --git a/apps/cli/tsup.config.ts b/apps/cli/tsup.config.ts index 2f51c7df..0d01471a 100644 --- a/apps/cli/tsup.config.ts +++ b/apps/cli/tsup.config.ts @@ -28,7 +28,7 @@ const { version } = JSON.parse( * metafile out of the package and removes their link trailers before anything is hashed or packed; the ordinary * closure guard refuses any map that a build outside prepack leaves in `dist`. */ -const CANDIDATE = process.env.RELAVIUM_CLI_CANDIDATE_STAGING !== undefined; +const CANDIDATE = process.env['RELAVIUM_CLI_CANDIDATE_STAGING'] !== undefined; const THIRD_PARTY_EXTERNAL = [ '@anthropic-ai/sdk', '@clack/prompts', From e8121aa453d19f991c919094cee7b5f8e1cec21b Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Wed, 4 Nov 2026 13:56:43 +0300 Subject: [PATCH 92/97] fix(repo): clear the release tooling's review and Sonar findings PR #96 review and SonarCloud findings in tools/: - The packed-manifest runtime dependency check is an allowlist (exact, caret or tilde registry versions) shared by the archive inspector and the receipt binding, so npm:, github:, git and tarball URLs, `*` and open ranges are refused; negative tests on both paths. - copyMatchesCommit asks Git for each copy's blob id (hash-object --no-filters, batched under the Windows command-line limit) instead of a plain SHA-1 of ours (S4790), keeps the ordinary-file proof that hash-object would follow a link past, and accepts the repository's object format; tests cover several batches, a linked copy and a short answer. - Every flagged sort states its code-unit order with one comparator (S2871 x24); localeCompare would reorder manifests and digests. - Fixtures resolve gpg, gpgconf and ps to absolute paths (S4036); the two maintainer/CI path arguments get an absolute-path check and a NOSONAR rationale (S8707). - Mechanical smells: S1121, S2094, S3358, S4043, S4123, S6353, S6557, S6582, S7755, S7763, S7780, S7781, S7786. Co-Authored-By: Claude Opus 5.5 --- tools/bundle-closure/closure.mjs | 13 +- tools/cli-package/archive.mjs | 34 ++++-- tools/cli-package/archive.test.mjs | 10 ++ tools/cli-package/binding.mjs | 10 +- tools/cli-package/binding.test.mjs | 3 + tools/cli-package/candidate-fixture.mjs | 10 +- tools/cli-package/candidate.mjs | 75 ++++++++---- tools/cli-package/candidate.test.mjs | 111 ++++++++++++++++++ tools/cli-package/consumer-lifetime.mjs | 2 +- tools/cli-package/container.mjs | 14 ++- .../cli-package/custody-fixtures/generate.mjs | 10 +- tools/cli-package/custody.mjs | 4 +- tools/cli-package/files.mjs | 12 ++ tools/cli-package/finalize.mjs | 11 +- tools/cli-package/frame-capture-keyring.mjs | 2 + tools/cli-package/frame-capture.mjs | 2 +- tools/cli-package/gpg-fixture.mjs | 11 +- tools/cli-package/manifest.mjs | 11 +- tools/cli-package/map-controls-core.mjs | 12 +- tools/cli-package/produce.mjs | 11 +- tools/cli-package/release-binding.mjs | 12 +- tools/cli-package/release-custody.mjs | 9 +- tools/cli-package/release-recipient.mjs | 4 +- tools/cli-package/seal.mjs | 10 +- tools/cli-package/smoke/index.mjs | 4 +- tools/cli-package/source-map.mjs | 2 +- tools/cli-package/symbolicate.mjs | 2 +- tools/cli-package/ustar-fixture.mjs | 4 +- 28 files changed, 324 insertions(+), 91 deletions(-) diff --git a/tools/bundle-closure/closure.mjs b/tools/bundle-closure/closure.mjs index f726f639..dfd25708 100644 --- a/tools/bundle-closure/closure.mjs +++ b/tools/bundle-closure/closure.mjs @@ -10,6 +10,12 @@ import { posix } from 'node:path'; import { isDeepStrictEqual } from 'node:util'; import ts from 'typescript'; +/** Code-unit order: deterministic and locale-independent, as `Array#sort` without a comparator. */ +function byCodeUnit(left, right) { + if (left < right) return -1; + return left > right ? 1 : 0; +} + export class BundleClosureError extends Error { constructor(code, detail) { super(`${code}: ${detail}`); @@ -223,7 +229,12 @@ function runtimeOutputs(outputs, directory, maps) { const entry = record(output, `map output ${path}`); bytes(entry.bytes, `map output ${path}.bytes`); if ( - !isDeepStrictEqual(Object.keys(entry).sort(), ['bytes', 'exports', 'imports', 'inputs']) || + !isDeepStrictEqual(Object.keys(entry).sort(byCodeUnit), [ + 'bytes', + 'exports', + 'imports', + 'inputs', + ]) || Object.keys(record(entry.inputs, `map output ${path}.inputs`)).length > 0 || strings(entry.exports, `map output ${path}.exports`).length > 0 || imports(entry.imports, `map output ${path}`).length > 0 diff --git a/tools/cli-package/archive.mjs b/tools/cli-package/archive.mjs index ee7b8741..b2e89ca5 100644 --- a/tools/cli-package/archive.mjs +++ b/tools/cli-package/archive.mjs @@ -80,6 +80,32 @@ function unpack(bytes) { } /** @param {string} path */ +/** + * A runtime dependency requirement the public manifest may carry: an exact, caret or tilde registry + * version, the form pnpm publishes for a `catalog:` entry. An allowlist rather than a list of refused + * protocols, so `npm:` aliases, `github:`, git and tarball URLs, `file:`/`link:`/`workspace:`/`catalog:` + * specifiers, `*` and open ranges never reach a consumer's install (a PR #96 review finding). + */ +const REGISTRY_REQUIREMENT = /^[~^]?\d{1,9}\.\d{1,9}\.\d{1,9}(?:-[0-9A-Za-z.-]{1,64})?$/u; + +/** + * Whether one public runtime dependency entry is admitted: an npm package name outside the private + * `@relavium/` scope, required at a registry version. Shared by the archive inspector and the package + * receipt binding so the two cannot drift. + * @param {string} name + * @param {unknown} requirement + * @returns {boolean} + */ +export function isPublicRuntimeDependency(name, requirement) { + return ( + /^(?:@[a-z0-9_.-]+\/)?[a-z0-9_.-]+$/u.test(name) && + name.length <= 200 && + !name.startsWith('@relavium/') && + typeof requirement === 'string' && + REGISTRY_REQUIREMENT.test(requirement) + ); +} + export function assertPublicPackagePath(path) { if ( !/^package\/(?:dist\/[A-Za-z0-9_-]+\.js|drizzle\/[0-9]{4}_[a-z0-9_]+\.sql|drizzle\/meta\/(?:[0-9]{4}_snapshot|_journal)\.json|package\.json|README\.md|CHANGELOG\.md|LICENSE)$/u.test( @@ -122,13 +148,7 @@ function manifest(bytes, expected) { if (dependencies === undefined) continue; if (!isRecord(dependencies)) refuse('MANIFEST_RUNTIME'); for (const [name, version] of Object.entries(dependencies)) { - if ( - name.startsWith('@relavium/') || - typeof version !== 'string' || - /^(?:workspace|catalog|file|link):/u.test(version) - ) { - refuse('MANIFEST_RUNTIME'); - } + if (!isPublicRuntimeDependency(name, version)) refuse('MANIFEST_RUNTIME'); } } return parsed; diff --git a/tools/cli-package/archive.test.mjs b/tools/cli-package/archive.test.mjs index 1566b793..ce9bf97b 100644 --- a/tools/cli-package/archive.test.mjs +++ b/tools/cli-package/archive.test.mjs @@ -296,6 +296,16 @@ test('manifest rejects runtime drift and unresolved/private runtime dependencies { commander: 'catalog:' }, { commander: 'workspace:*' }, { '@relavium/core': '0.0.0' }, + // A PR #96 review: a denylist admitted every other non-registry specifier. + { commander: 'npm:evil@1.0.0' }, + { commander: 'github:attacker/commander' }, + { commander: 'git+https://example.invalid/commander.git' }, + { commander: 'https://example.invalid/commander-12.1.0.tgz' }, + { commander: '*' }, + { commander: '>=12.0.0' }, + { commander: 'latest' }, + { commander: '' }, + { commander: 12 }, ]) { const badPackage = { ...expectedPackage, dependencies }; const files = inventory(); diff --git a/tools/cli-package/binding.mjs b/tools/cli-package/binding.mjs index b0c7dc2a..928dc82b 100644 --- a/tools/cli-package/binding.mjs +++ b/tools/cli-package/binding.mjs @@ -1,6 +1,6 @@ import { isDeepStrictEqual } from 'node:util'; import { BOOTSTRAP_CONTROL_COUNT } from '../cli-bootstrap/inventory.mjs'; -import { ARCHIVE_LIMITS, assertPublicPackagePath } from './archive.mjs'; +import { ARCHIVE_LIMITS, assertPublicPackagePath, isPublicRuntimeDependency } from './archive.mjs'; import { REQUIRED_SMOKE_CASES } from './smoke/cases.mjs'; /** @typedef {{kind: 'local'} | {kind: 'github', repository: string, workflow: string, runId: string, runAttempt: string, event: string, source: string}} Origin */ @@ -218,13 +218,7 @@ function publicPackage(value) { refuse('PACKAGE'); for (const [name, requirement] of Object.entries(dependencies)) { string(name, /^(?:@[a-z0-9_.-]+\/)?[a-z0-9_.-]+$/u, 200); - if ( - name.startsWith('@relavium/') || - typeof requirement !== 'string' || - requirement.length > 100 || - /(?:workspace|catalog|file|link):/u.test(requirement) - ) - refuse('PACKAGE'); + if (!isPublicRuntimeDependency(name, requirement)) refuse('PACKAGE'); } // Full package metadata is public npm material. Keep it bounded; private map // manifests and build/source inventories have no admitted top-level field. diff --git a/tools/cli-package/binding.test.mjs b/tools/cli-package/binding.test.mjs index def5fc98..06660761 100644 --- a/tools/cli-package/binding.test.mjs +++ b/tools/cli-package/binding.test.mjs @@ -186,6 +186,9 @@ test('package metadata is public-only and keeps private packages out of runtime' { dependencies: { '@relavium/core': '0.0.0' } }, { dependencies: { commander: 'workspace:*' } }, { dependencies: { commander: 'catalog:' } }, + ...['npm:evil@1.0.0', 'github:attacker/commander', 'https://example.invalid/c.tgz', '*'].map( + (requirement) => ({ dependencies: { commander: requirement } }), + ), ]) refuses(() => validatePackageReceipt({ ...pack, package: { ...pack.package, ...changed } }, expected), diff --git a/tools/cli-package/candidate-fixture.mjs b/tools/cli-package/candidate-fixture.mjs index 27075164..f95deaec 100644 --- a/tools/cli-package/candidate-fixture.mjs +++ b/tools/cli-package/candidate-fixture.mjs @@ -40,7 +40,10 @@ export const posixModes = process.platform !== 'win32'; export const PRELOAD = fileURLToPath(new URL('./frame-capture.mjs', import.meta.url)); let count = 0; /** A fresh scratch directory name. */ -export const fresh = (name) => join(scratch, `${name}-${(count += 1)}`); +export const fresh = (name) => { + count += 1; + return join(scratch, `${name}-${count}`); +}; export const SOURCES = { 'packages/core/src/yaml-decode.ts': [ @@ -138,7 +141,8 @@ export function put(root, path, bytes) { export async function candidate({ sources = SOURCES, parent = scratch } = {}) { const root = fresh('candidate'); const staging = fresh('staging'); - const work = join(parent, `work-${(count += 1)}`); + count += 1; + const work = join(parent, `work-${count}`); for (const directory of [staging, work]) { mkdirSync(directory, { recursive: true }); chmodSync(directory, 0o700); @@ -186,7 +190,7 @@ export async function candidate({ sources = SOURCES, parent = scratch } = {}) { const outputs = []; const maps = []; for (const file of built.outputFiles) { - const name = posix.basename(file.path.split('\\').join('/')); + const name = posix.basename(file.path.replaceAll('\\', '/')); if (name.endsWith('.map')) { const map = JSON.parse(file.text); map.sources = map.sources.map((source) => diff --git a/tools/cli-package/candidate.mjs b/tools/cli-package/candidate.mjs index 8c32b8f7..c0a70db3 100644 --- a/tools/cli-package/candidate.mjs +++ b/tools/cli-package/candidate.mjs @@ -7,7 +7,6 @@ * output, and no build cache is read or written. */ import assert from 'node:assert/strict'; -import { createHash } from 'node:crypto'; import { chmodSync, existsSync, @@ -21,7 +20,7 @@ import { import { dirname, isAbsolute, join, resolve } from 'node:path'; import { isDeepStrictEqual } from 'node:util'; import { ENGINE_PACKAGES } from './finalize.mjs'; -import { readOrdinaryFile } from './files.mjs'; +import { readOrdinaryFile, byCodeUnit } from './files.mjs'; import { within } from './paths.mjs'; import { GIT_NULL_DEVICE, @@ -76,7 +75,7 @@ const GIT_CONFIG_VARIABLES = Object.freeze([ 'SystemRoot', 'WINDIR', ]); -const GIT_CONFIG_ENTRY = /^GIT_CONFIG_(?:KEY|VALUE)_(?:0|[1-9][0-9]{0,3})$/u; +const GIT_CONFIG_ENTRY = /^GIT_CONFIG_(?:KEY|VALUE)_(?:0|[1-9]\d{0,3})$/u; /** * The developer's global Git excludes file, so that the candidate's isolated Git ignores what the @@ -161,7 +160,7 @@ function readArguments(excludes) { */ function uncheckedPaths(output) { return gitPaths(output) - .filter((entry) => entry[0] === 'S' || /^[a-z]$/u.test(entry[0])) + .filter((entry) => entry.startsWith('S') || /^[a-z]$/u.test(entry[0])) .map((entry) => entry.slice(2)); } @@ -229,7 +228,7 @@ export async function candidateFiles( const deleted = new Set( gitPaths((await run('git', ['ls-files', '-z', '--deleted'], options)).stdout), ); - const files = [...new Set(listed)].filter((path) => !deleted.has(path)).sort(); + const files = [...new Set(listed)].filter((path) => !deleted.has(path)).sort(byCodeUnit); assert.ok(files.length > 0 && files.length <= limits.files, 'PACKED_CLI_CANDIDATE_FILES'); return files; } @@ -319,9 +318,10 @@ export async function dependencyStore({ repository, environment, home, pnpm = ru /** * Whether every copied file is byte for byte the commit's blob. A clean status does not prove it: * line-ending conversion (`core.autocrlf`, the Git for Windows default) and Git's stat cache can report - * converted working-tree bytes as unchanged, and the candidate is built from those bytes. Each copy's - * Git blob id (SHA-1 of the blob header and its raw bytes, no filter) must equal the commit tree's - * entry for its path. + * converted working-tree bytes as unchanged, and the candidate is built from those bytes. Each copy, + * an ordinary file, must have the commit tree's blob id for its path, as Git itself names its raw bytes + * (`hash-object --no-filters`): Git's collision-detecting SHA-1 in the repository's object format, not a + * plain hash of ours. * @param {{repository: string, candidate: string, source: string, files: readonly string[], * run?: typeof runCommand}} options */ @@ -332,22 +332,55 @@ export async function copyMatchesCommit({ files, run = runCommand, }) { - const listing = ( - await run('git', ['--no-replace-objects', 'ls-tree', '-r', '-z', '--full-tree', source], { - cwd: repository, - env: isolatedGitEnvironment(), - }) - ).stdout; + const git = async (args) => + ( + await run('git', ['--no-replace-objects', ...args], { + cwd: repository, + env: isolatedGitEnvironment(), + }) + ).stdout; const blobs = new Map(); - for (const entry of gitPaths(listing)) { - const match = /^[0-7]{6} blob ([0-9a-f]{40})\t(.+)$/su.exec(entry); + for (const entry of gitPaths(await git(['ls-tree', '-r', '-z', '--full-tree', source]))) { + const match = /^[0-7]{6} blob ([0-9a-f]{40,64})\t(.+)$/su.exec(entry); if (match !== null) blobs.set(match[2], match[1]); } - return files.every((path) => { - const bytes = readOrdinaryFile(join(candidate, ...path.split('/')), CANDIDATE_LIMITS.fileBytes); - const id = createHash('sha1').update(`blob ${bytes.length}\0`).update(bytes).digest('hex'); - return blobs.get(path) === id; - }); + const copies = files.map((path) => join(candidate, ...path.split('/'))); + // `hash-object` follows a symbolic link, so each copy is first proven an ordinary, bounded file. + for (const copy of copies) readOrdinaryFile(copy, CANDIDATE_LIMITS.fileBytes); + for (const batch of hashObjectBatches(copies)) { + const ids = await git(['hash-object', '--no-filters', '--', ...batch.paths]); + const names = ids.split('\n').filter((line) => line !== ''); + if (names.length !== batch.paths.length) return false; + for (const [offset, id] of names.entries()) { + if (blobs.get(files[batch.start + offset]) !== id) return false; + } + } + return true; +} + +/** Windows caps one command line at 32,767 characters; a batch of paths stays well inside it. */ +const HASH_OBJECT_BATCH_CHARACTERS = 16 * 1024; + +/** + * Consecutive batches of paths for one `git hash-object` each, with the index of each batch's first path. + * @param {readonly string[]} paths + * @returns {{start: number, paths: string[]}[]} + */ +function hashObjectBatches(paths) { + const batches = []; + let current = { start: 0, paths: [] }; + let characters = 0; + for (const [index, path] of paths.entries()) { + if (current.paths.length > 0 && characters + path.length + 1 > HASH_OBJECT_BATCH_CHARACTERS) { + batches.push(current); + current = { start: index, paths: [] }; + characters = 0; + } + current.paths.push(path); + characters += path.length + 1; + } + if (current.paths.length > 0) batches.push(current); + return batches; } /** diff --git a/tools/cli-package/candidate.test.mjs b/tools/cli-package/candidate.test.mjs index 31a0f498..4f73f0af 100644 --- a/tools/cli-package/candidate.test.mjs +++ b/tools/cli-package/candidate.test.mjs @@ -875,6 +875,23 @@ function preparation({ .join('\0'), }; } + // Git names each copy's raw bytes, asked of the original checkout like the listing. + if (args[1] === 'hash-object') { + assert.deepEqual(args.slice(0, 4), [ + '--no-replace-objects', + 'hash-object', + '--no-filters', + '--', + ]); + assert.equal(options.cwd, repository); + assert.deepEqual(options.env, isolatedGitEnvironment()); + return { + stdout: args + .slice(4) + .map((path) => `${blobId(readFileSync(path))}\n`) + .join(''), + }; + } throw new Error(`unexpected git ${key}`); }; let installOptions; @@ -937,6 +954,20 @@ test('candidate preparation copies, re-checks identity, installs and verifies li `git ${READ()}status --porcelain=v1 -z --untracked-files=all`, 'git ls-files -v -z', `git --no-replace-objects ls-tree -r -z --full-tree ${SHA}`, + `git --no-replace-objects hash-object --no-filters -- ${[ + 'package.json', + 'core', + 'db', + 'llm', + 'mcp', + 'shared', + ] + .map((name) => + name === 'package.json' + ? join(ports.candidate, name) + : join(ports.candidate, 'packages', name, 'package.json'), + ) + .join(' ')}`, 'pnpm store path', `pnpm install --offline --frozen-lockfile --ignore-scripts --store-dir ${ports.store}`, ]); @@ -1016,6 +1047,86 @@ test('a checkout whose line endings Git converts copies bytes that are not the c ); }); +test('every hash-object batch is compared, the last included', async () => { + const { repository, inRepository } = await committedRepository('hash-object-batches'); + // Enough long paths for several 16 KiB command lines, committed, then copied unchanged. + const files = Array.from( + { length: 240 }, + (_, index) => `nested/${String(index).padStart(3, '0')}-${'n'.repeat(90)}.ts`, + ); + for (const path of files) + write(join(repository, ...path.split('/')), `export const v = '${path}';\n`); + await inRepository(['add', 'nested']); + await inRepository([ + '-c', + 'user.name=t', + '-c', + 'user.email=t@example.invalid', + 'commit', + '-qm', + 'two', + ]); + const identity = await sourceIdentity(repository); + const candidate = fresh('batched-copy'); + copyCandidateTree({ repository, destination: candidate, files }); + const calls = []; + const run = (command, args, options) => { + if (args[1] === 'hash-object') calls.push(args.length - 4); + return runCommand(command, args, options); + }; + const matches = () => + copyMatchesCommit({ repository, candidate, source: identity.source, files, run }); + assert.equal(await matches(), true); + assert.ok(calls.length >= 2, `expected several batches, saw ${calls.length}`); + assert.equal( + calls.reduce((sum, count) => sum + count, 0), + files.length, + ); + // A change in the final batch's last file is still seen. + writeFileSync(join(candidate, ...files.at(-1).split('/')), 'export const v = 0;\n'); + assert.equal(await matches(), false); +}); + +test( + 'a copy that is a symbolic link to the committed bytes is refused, not hashed through', + { + skip: !posixModes, + }, + async () => { + const { repository } = await committedRepository('linked-copy'); + const identity = await sourceIdentity(repository); + const candidate = fresh('linked-copy-candidate'); + const elsewhere = join(fresh('linked-copy-target'), 'kept.ts'); + write(elsewhere, 'export {};\n'); + symlinkSync(elsewhere, join(candidate, 'kept.ts')); + await assert.rejects( + copyMatchesCommit({ repository, candidate, source: identity.source, files: ['kept.ts'] }), + /PACKED_CLI_FILE_KIND_OR_BOUND/, + ); + }, +); + +test('a hash-object answer missing a line is not a match', async () => { + const { repository } = await committedRepository('short-answer'); + const identity = await sourceIdentity(repository); + const candidate = fresh('short-answer-candidate'); + copyCandidateTree({ repository, destination: candidate, files: ['kept.ts'] }); + const run = async (command, args, options) => { + const result = await runCommand(command, args, options); + return args[1] === 'hash-object' ? { ...result, stdout: '' } : result; + }; + assert.equal( + await copyMatchesCommit({ + repository, + candidate, + source: identity.source, + files: ['kept.ts'], + run, + }), + false, + ); +}); + test('a store the caller names reaches the install, and pnpm is never asked to discover one', async () => { const ports = preparation(); const explicit = fresh('explicit-store'); diff --git a/tools/cli-package/consumer-lifetime.mjs b/tools/cli-package/consumer-lifetime.mjs index 5a251dab..6b11bab7 100644 --- a/tools/cli-package/consumer-lifetime.mjs +++ b/tools/cli-package/consumer-lifetime.mjs @@ -49,7 +49,7 @@ export async function completeConsumer({ exercise, cleanup, cleanupFailed, publi primaryFailure = true; } try { - await cleanup(); + cleanup(); // synchronous by contract, see above } catch (error) { if (!primaryFailure) throw error; secondary(); diff --git a/tools/cli-package/container.mjs b/tools/cli-package/container.mjs index f17f57da..1b5bb159 100644 --- a/tools/cli-package/container.mjs +++ b/tools/cli-package/container.mjs @@ -11,7 +11,7 @@ import { lstatSync, readdirSync } from 'node:fs'; import { join, posix } from 'node:path'; import { isDeepStrictEqual } from 'node:util'; import { CustodyError, refuseCustody } from './custody-error.mjs'; -import { readOrdinaryFile, sha256 } from './files.mjs'; +import { readOrdinaryFile, sha256, byCodeUnit } from './files.mjs'; import { MANIFEST_FILE, RECORD_BYTES, validateManifest } from './manifest.mjs'; import { readUstar, writeUstar } from './ustar.mjs'; @@ -50,7 +50,7 @@ function recorded(list) { !list.every( (item) => isRecord(item) && - isDeepStrictEqual(Object.keys(item).sort(), ['bytes', 'path', 'sha256']) && + isDeepStrictEqual(Object.keys(item).sort(byCodeUnit), ['bytes', 'path', 'sha256']) && typeof item.path === 'string' && Number.isSafeInteger(item.bytes) && typeof item.sha256 === 'string', @@ -86,8 +86,8 @@ function checkContents(files) { const maps = manifest.outputs.map((output) => `maps/${posix.basename(output.map.path)}`); if ( !isDeepStrictEqual( - [...files.keys()].filter((path) => !RECORDS.includes(path)).sort(), - [...maps].sort(), + [...files.keys()].filter((path) => !RECORDS.includes(path)).sort(byCodeUnit), + [...maps].sort(byCodeUnit), ) ) { refuseCustody('CONTAINER_SET'); @@ -104,7 +104,7 @@ function checkContents(files) { const record = json(/** @type {Buffer} */ (files.get('finalized.json')), 'CONTAINER_RECORD'); if ( !isRecord(record) || - !isDeepStrictEqual(Object.keys(record).sort(), [ + !isDeepStrictEqual(Object.keys(record).sort(byCodeUnit), [ 'maps', 'outputs', 'upstreamMaps', @@ -172,7 +172,9 @@ function build(staging, limits) { return refuseCustody('CONTAINER_ENTRY'); } }; - if (!isDeepStrictEqual(readdirSync(staging).sort(), [...RECORDS, 'maps'].sort())) { + if ( + !isDeepStrictEqual(readdirSync(staging).sort(byCodeUnit), [...RECORDS, 'maps'].sort(byCodeUnit)) + ) { refuseCustody('CONTAINER_SET'); } // lstat: a linked maps directory is not a directory. diff --git a/tools/cli-package/custody-fixtures/generate.mjs b/tools/cli-package/custody-fixtures/generate.mjs index 373bc577..38ac5f59 100644 --- a/tools/cli-package/custody-fixtures/generate.mjs +++ b/tools/cli-package/custody-fixtures/generate.mjs @@ -13,17 +13,20 @@ import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'n import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { resolveGpg } from '../gnupg.mjs'; const here = fileURLToPath(new URL('.', import.meta.url)); // A short home: the agent's socket path must fit the platform's limit (104 bytes on macOS). const home = mkdtempSync(join(tmpdir(), 'rg-')); +// The installation from absolute PATH entries only, as the release path resolves it. +const installation = resolveGpg(); const PAST = '20200101T000000!'; const NO_AEAD = 'AES256 AES192 AES SHA512 SHA384 SHA256 ZLIB ZIP Uncompressed'; /** @param {string[]} args @param {{input?: string, past?: boolean}} [options] */ function gpg(args, { input, past = false } = {}) { const run = spawnSync( - 'gpg', + installation.command, [ '--homedir', home, @@ -149,7 +152,8 @@ try { assert.ok(!readFileSync(join(here, name), 'utf8').includes('PRIVATE KEY'), name); } } finally { - spawnSync('gpgconf', ['--homedir', home, '--kill', 'all'], { stdio: 'ignore' }); - spawnSync('gpgconf', ['--homedir', home, '--remove-socketdir'], { stdio: 'ignore' }); + const gpgconf = installation.gpgconf.command; + spawnSync(gpgconf, ['--homedir', home, '--kill', 'all'], { stdio: 'ignore' }); + spawnSync(gpgconf, ['--homedir', home, '--remove-socketdir'], { stdio: 'ignore' }); rmSync(home, { recursive: true, force: true }); } diff --git a/tools/cli-package/custody.mjs b/tools/cli-package/custody.mjs index 77b85533..138e4a43 100644 --- a/tools/cli-package/custody.mjs +++ b/tools/cli-package/custody.mjs @@ -23,7 +23,7 @@ import { dirname, join } from 'node:path'; import { isDeepStrictEqual } from 'node:util'; import { readPrivateContainer } from './container.mjs'; import { CustodyError, refuseCustody } from './custody-error.mjs'; -import { readOrdinaryFile, sha256 } from './files.mjs'; +import { readOrdinaryFile, sha256, byCodeUnit } from './files.mjs'; import { writePrivateFile } from './finalize.mjs'; import { GNUPG_CALL, @@ -74,7 +74,7 @@ export function parsePins(pins) { if (pins === null || typeof pins !== 'object') return refuseCustody('PIN_FORMAT'); const { primary, subkey } = /** @type {Record} */ (pins); if ( - !isDeepStrictEqual(Object.keys(pins).sort(), ['primary', 'subkey']) || + !isDeepStrictEqual(Object.keys(pins).sort(byCodeUnit), ['primary', 'subkey']) || typeof primary !== 'string' || typeof subkey !== 'string' || !FINGERPRINT.test(primary) || diff --git a/tools/cli-package/files.mjs b/tools/cli-package/files.mjs index e63455ef..6d44f2dd 100644 --- a/tools/cli-package/files.mjs +++ b/tools/cli-package/files.mjs @@ -14,6 +14,18 @@ import { join } from 'node:path'; export const sha256 = (bytes) => createHash('sha256').update(bytes).digest('hex'); +/** + * Code-unit order for a string sort: deterministic and locale-independent, exactly what `Array#sort` does + * without a comparator. Manifests, inventories and digests depend on that order, so `localeCompare` would + * be wrong here; the explicit comparator only states it. + * @param {string} left + * @param {string} right + */ +export function byCodeUnit(left, right) { + if (left < right) return -1; + return left > right ? 1 : 0; +} + export function readOrdinaryFile(path, maxBytes) { assert.ok(Number.isSafeInteger(maxBytes) && maxBytes > 0); const info = lstatSync(path, { bigint: true }); diff --git a/tools/cli-package/finalize.mjs b/tools/cli-package/finalize.mjs index 884cb728..37ae64ad 100644 --- a/tools/cli-package/finalize.mjs +++ b/tools/cli-package/finalize.mjs @@ -33,7 +33,7 @@ import { basename, isAbsolute, join, posix } from 'node:path'; import { TextDecoder, isDeepStrictEqual } from 'node:util'; import { checkBundleClosure } from '../bundle-closure/closure.mjs'; import { MODE_EVIDENCE, emittedInventory } from '../bundle-closure/inventory.mjs'; -import { flatFiles, readOrdinaryFile, sha256 } from './files.mjs'; +import { flatFiles, readOrdinaryFile, sha256, byCodeUnit } from './files.mjs'; import { within } from './paths.mjs'; import { SourceMapError, @@ -236,7 +236,7 @@ function upstreamSources(repository, packageName, jsPath, texts) { const map = readJson(onDisk(repository, mapPath), LIMITS.upstreamMapBytes, mapPath); if ( !isRecord(map) || - !isDeepStrictEqual(Object.keys(map).sort(), [ + !isDeepStrictEqual(Object.keys(map).sort(byCodeUnit), [ 'file', 'mappings', 'names', @@ -362,7 +362,7 @@ function finalMap(repository, jsName, map, expected, code, texts) { ), ); if (new Set(sources).size !== sources.length) refuse('MAP_SOURCE_DUPLICATE', subject); - if (!isDeepStrictEqual([...sources].sort(), [...expected].sort())) { + if (!isDeepStrictEqual([...sources].sort(byCodeUnit), [...expected].sort(byCodeUnit))) { refuse('MAP_SOURCE_COVERAGE', subject); } checkMappings(repository, map, code, sources, texts, 'MAP', subject); @@ -466,7 +466,10 @@ export function finalizeCandidate({ repository, staging }) { const entries = readdirSync(dist, { withFileTypes: true }); if ( entries.some((entry) => !entry.isFile()) || - !isDeepStrictEqual(entries.map((entry) => entry.name).sort(), expectedEntries.sort()) + !isDeepStrictEqual( + entries.map((entry) => entry.name).sort(byCodeUnit), + [...expectedEntries].sort(byCodeUnit), + ) ) { refuse('OUTPUT_INVENTORY'); } diff --git a/tools/cli-package/frame-capture-keyring.mjs b/tools/cli-package/frame-capture-keyring.mjs index 7c4a38ca..d4b74c07 100644 --- a/tools/cli-package/frame-capture-keyring.mjs +++ b/tools/cli-package/frame-capture-keyring.mjs @@ -3,7 +3,9 @@ const refused = () => { throw new Error('The symbolication run refuses the keyring'); }; +// A class, not a function, so the exports below extend it as the real package's classes do. export class Entry { + // NOSONAR — see above constructor() { refused(); } diff --git a/tools/cli-package/frame-capture.mjs b/tools/cli-package/frame-capture.mjs index 6fb817e3..2cd7a54d 100644 --- a/tools/cli-package/frame-capture.mjs +++ b/tools/cli-package/frame-capture.mjs @@ -19,7 +19,7 @@ const manifest = process.env.RELAVIUM_FRAME_CAPTURE_PACKAGE; const canary = process.env.RELAVIUM_FRAME_CAPTURE_CANARY; if (!log || !manifest || !canary) throw new Error('The symbolication preload is not configured'); if (typeof nodeModule.registerHooks !== 'function') { - throw new Error('The symbolication preload needs module.registerHooks'); + throw new TypeError('The symbolication preload needs module.registerHooks'); } // The candidate root holds `apps/cli/package.json`; pnpm installs the CLI's dependencies below it. diff --git a/tools/cli-package/gpg-fixture.mjs b/tools/cli-package/gpg-fixture.mjs index 12b4c18f..d654f2a5 100644 --- a/tools/cli-package/gpg-fixture.mjs +++ b/tools/cli-package/gpg-fixture.mjs @@ -26,8 +26,9 @@ const SOCKET = '/S.gpg-agent.browser'; /** Every process whose command line names `home`. */ export function processesNaming(home) { + // An absolute path, not a PATH lookup; `/bin/ps` exists on macOS and on Linux (merged-/usr or not). const listing = spawnSync( - 'ps', + '/bin/ps', process.platform === 'darwin' ? ['-axo', 'command'] : ['-eo', 'args'], { encoding: 'utf8' }, ); @@ -70,8 +71,12 @@ export function ephemeralKey() { return run.stdout; }; const dispose = () => { - spawnSync('gpgconf', ['--homedir', home, '--kill', 'all'], { stdio: 'ignore', env }); - spawnSync('gpgconf', ['--homedir', home, '--remove-socketdir'], { stdio: 'ignore', env }); + // The `gpgconf` of the same installation, not a PATH lookup. + spawnSync(gpg.gpgconf.command, ['--homedir', home, '--kill', 'all'], { stdio: 'ignore', env }); + spawnSync(gpg.gpgconf.command, ['--homedir', home, '--remove-socketdir'], { + stdio: 'ignore', + env, + }); rmSync(home, { recursive: true, force: true }); rmSync(exportDirectory, { recursive: true, force: true }); }; diff --git a/tools/cli-package/manifest.mjs b/tools/cli-package/manifest.mjs index b9a91af0..f094b099 100644 --- a/tools/cli-package/manifest.mjs +++ b/tools/cli-package/manifest.mjs @@ -13,7 +13,7 @@ import { basename, join } from 'node:path'; import { isDeepStrictEqual } from 'node:util'; import { isPackageVersion } from './binding.mjs'; import { ENGINE_PACKAGES, writePrivateFile } from './finalize.mjs'; -import { readOrdinaryFile, sha256 } from './files.mjs'; +import { readOrdinaryFile, sha256, byCodeUnit } from './files.mjs'; import { isolatedGitEnvironment, runCommand } from './process.mjs'; export const MANIFEST_FILE = 'manifest.json'; @@ -45,7 +45,7 @@ const OUTPUT = /^dist\/[A-Za-z0-9_-]+\.js$/u; const SEGMENT = '[A-Za-z0-9_-][A-Za-z0-9_.-]*'; /** A repository-relative original source the finalizer admits: CLI or engine TypeScript. */ const SOURCE = new RegExp( - `^(?:apps/cli|packages/(?:${ENGINE_PACKAGES.join('|')}))/src/(?:${SEGMENT}/)*${SEGMENT}\\.tsx?$`, + String.raw`^(?:apps/cli|packages/(?:${ENGINE_PACKAGES.join('|')}))/src/(?:${SEGMENT}/)*${SEGMENT}\.tsx?$`, 'u', ); const VERSION = /^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]{1,64})?$/u; @@ -105,7 +105,8 @@ function isRecord(value) { */ export function validateManifest(value) { const exact = (record, keys) => - isRecord(record) && isDeepStrictEqual(Object.keys(record).sort(), [...keys].sort()); + isRecord(record) && + isDeepStrictEqual(Object.keys(record).sort(byCodeUnit), [...keys].sort(byCodeUnit)); if ( !exact(value, [ 'version', @@ -235,7 +236,7 @@ export async function sealPrivateManifest({ const sources = Object.fromEntries( [...named] .map(String) - .sort() + .sort(byCodeUnit) .map((name) => { if (!SOURCE.test(name)) refuse('SOURCE_PATH'); return [name, sha256(read(candidate, name, 8 * 1024 * 1024))]; @@ -311,7 +312,7 @@ export function verifySealedDist({ candidate, manifest, tarball }) { } catch { refuse('DIST_DRIFT'); } - const expected = manifest.outputs.map((output) => basename(output.path)).sort(); + const expected = manifest.outputs.map((output) => basename(output.path)).sort(byCodeUnit); if (!isDeepStrictEqual(names, expected)) refuse('DIST_DRIFT'); for (const output of manifest.outputs) { let bytes; diff --git a/tools/cli-package/map-controls-core.mjs b/tools/cli-package/map-controls-core.mjs index 6de4a20d..84606d0f 100644 --- a/tools/cli-package/map-controls-core.mjs +++ b/tools/cli-package/map-controls-core.mjs @@ -32,7 +32,7 @@ import { basename, dirname, join } from 'node:path'; import { originFromEnvironment } from './binding.mjs'; import { sourceIdentity } from './candidate.mjs'; import { CANDIDATE_STAGING_ENV } from './finalize.mjs'; -import { readOrdinaryFile, sha256 } from './files.mjs'; +import { readOrdinaryFile, sha256, byCodeUnit } from './files.mjs'; import { readPrivateManifest, sealPrivateManifest, verifySealedDist } from './manifest.mjs'; import { emitFreshPackage } from './pipeline.mjs'; import { producePackage } from './produce.mjs'; @@ -218,8 +218,10 @@ async function runControls(context, state) { const record = (id, expected, observed) => state.results.push({ id, expected, observed }); const controls = privateDirectory(join(staging, 'controls')); let count = 0; - const scratch = (id) => - privateDirectory(join(controls, `${String((count += 1)).padStart(2, '0')}-${id}`)); + const scratch = (id) => { + count += 1; + return privateDirectory(join(controls, `${String(count).padStart(2, '0')}-${id}`)); + }; const cli = join(candidate, 'apps', 'cli'); const tarball = join(artifact, receipt.tarball); const sealed = readPrivateManifest(staging); @@ -283,8 +285,8 @@ async function runControls(context, state) { [...finalized.outputs, ...finalized.maps] .filter((entry) => sealedDigests.get(entry.path) !== entry.sha256) .map((entry) => entry.path) - .sort(), - [...changed].sort(), + .sort(byCodeUnit), + [...changed].sort(byCodeUnit), 'PACKED_CLI_MAP_CONTROL_TARGET', ); writeFileSync(join(copy, 'finalized.json'), JSON.stringify(finalized), { diff --git a/tools/cli-package/produce.mjs b/tools/cli-package/produce.mjs index 3a625d0c..86244c5d 100644 --- a/tools/cli-package/produce.mjs +++ b/tools/cli-package/produce.mjs @@ -19,7 +19,7 @@ import { writeSync, } from 'node:fs'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { isAbsolute, join } from 'node:path'; import { assertNoBuildCache, globalExcludesFile, prepareCandidate } from './candidate.mjs'; import { within } from './paths.mjs'; import { exitOnInterrupt, isolatedEnvironment, runPnpm } from './process.mjs'; @@ -67,8 +67,13 @@ export async function producePackage({ } = ports; // The seal's symbolication needs Node 22.15 or later; refuse an older producer before anything exists. assert.ok(capturable(), 'PACKED_CLI_PRODUCER_NODE'); - mkdirSync(destination, { recursive: true }); - assert.deepEqual(readdirSync(destination), [], 'PACKED_CLI_FRESH_DESTINATION_REQUIRED'); + // The artifact directory is the maintainer's or CI's argument and may be anywhere; it must be + // absolute (pack.mjs resolves it) and empty. No model supplies it, which is what Sonar's S8707 + // taint rule assumes of a command-line argument. + assert.ok(isAbsolute(destination), 'PACKED_CLI_FRESH_DESTINATION_REQUIRED'); + mkdirSync(destination, { recursive: true }); // NOSONAR — a maintainer/CI path, see above + const present = readdirSync(destination); // NOSONAR — a maintainer/CI path, see above + assert.deepEqual(present, [], 'PACKED_CLI_FRESH_DESTINATION_REQUIRED'); const home = realpathSync.native(mkdtempSync(join(temporary, 'relavium-package-build-'))); let root; /** Remove each of `paths`, attempting every one; the first failure, if any. */ diff --git a/tools/cli-package/release-binding.mjs b/tools/cli-package/release-binding.mjs index 712384c8..b01d4824 100644 --- a/tools/cli-package/release-binding.mjs +++ b/tools/cli-package/release-binding.mjs @@ -13,7 +13,7 @@ import { join } from 'node:path'; import { isDeepStrictEqual, TextDecoder } from 'node:util'; import { ARCHIVE_LIMITS } from './archive.mjs'; import { CustodyError, refuseCustody } from './custody-error.mjs'; -import { readOrdinaryFile, sha256 } from './files.mjs'; +import { readOrdinaryFile, sha256, byCodeUnit } from './files.mjs'; import { isProductionRecipient, isSyntheticSource } from './release-recipient.mjs'; /** The receipt's file name beside the ciphertext, as the interim release guard keys on it. */ @@ -61,7 +61,10 @@ const isRecord = (value) => value !== null && typeof value === 'object' && !Arra /** @param {unknown} value @param {readonly string[]} keys */ function exact(value, keys) { - if (!isRecord(value) || !isDeepStrictEqual(Object.keys(value).sort(), [...keys].sort())) { + if ( + !isRecord(value) || + !isDeepStrictEqual(Object.keys(value).sort(byCodeUnit), [...keys].sort(byCodeUnit)) + ) { refuseCustody('BINDING_SHAPE'); } return /** @type {Record} */ (value); @@ -332,8 +335,9 @@ function verify(directory, expected) { RELEASE_BINDING_FILE, tarballName(packageVersion), ciphertextFileName(packageVersion), - ].sort(); - if (!isDeepStrictEqual(readdirSync(directory).sort(), names)) refuseCustody('CANDIDATE_ENTRIES'); + ].sort(byCodeUnit); + if (!isDeepStrictEqual(readdirSync(directory).sort(byCodeUnit), names)) + refuseCustody('CANDIDATE_ENTRIES'); let bytes; try { bytes = readOrdinaryFile(join(directory, RELEASE_BINDING_FILE), RECEIPT_BYTES); diff --git a/tools/cli-package/release-custody.mjs b/tools/cli-package/release-custody.mjs index a9223447..97347f93 100644 --- a/tools/cli-package/release-custody.mjs +++ b/tools/cli-package/release-custody.mjs @@ -25,7 +25,7 @@ import { ARCHIVE_LIMITS } from './archive.mjs'; import { readPrivateContainer } from './container.mjs'; import { CustodyError, refuseCustody } from './custody-error.mjs'; import { encryptOnce } from './custody.mjs'; -import { readOrdinaryFile, sha256 } from './files.mjs'; +import { readOrdinaryFile, sha256, byCodeUnit } from './files.mjs'; import { RELEASE_BINDING_FILE, RELEASE_BINDING_KIND, @@ -224,7 +224,12 @@ function seal(record, origin, write) { // An existing receipt refuses before anything is read; one is never written over. if (present(join(destination, RELEASE_BINDING_FILE))) refuseCustody('ALREADY_SEALED'); const tarball = tarballName(manifest.packageVersion); - if (!isDeepStrictEqual(readdirSync(destination).sort(), [record.name, tarball].sort())) { + if ( + !isDeepStrictEqual( + readdirSync(destination).sort(byCodeUnit), + [record.name, tarball].sort(byCodeUnit), + ) + ) { refuseCustody('CANDIDATE_ENTRIES'); } // The ciphertext must still be the bytes encrypted: the receipt is sealed only after it exists. diff --git a/tools/cli-package/release-recipient.mjs b/tools/cli-package/release-recipient.mjs index 6ab47496..adaae979 100644 --- a/tools/cli-package/release-recipient.mjs +++ b/tools/cli-package/release-recipient.mjs @@ -13,7 +13,7 @@ import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { isDeepStrictEqual } from 'node:util'; import { CustodyError, refuseCustody } from './custody-error.mjs'; -import { readOrdinaryFile } from './files.mjs'; +import { readOrdinaryFile, byCodeUnit } from './files.mjs'; /** The committed production recipient directory, without a trailing separator so a link is never followed. */ export const PRODUCTION_RECIPIENT_ROOT = fileURLToPath( @@ -108,7 +108,7 @@ function fixtureFingerprints(file) { !entries.every( (entry) => isRecord(entry) && - isDeepStrictEqual(Object.keys(entry).sort(), ['primary', 'subkey']) && + isDeepStrictEqual(Object.keys(entry).sort(byCodeUnit), ['primary', 'subkey']) && isFingerprint(entry.primary) && isFingerprint(entry.subkey), ) diff --git a/tools/cli-package/seal.mjs b/tools/cli-package/seal.mjs index ebac284c..9837d8cc 100644 --- a/tools/cli-package/seal.mjs +++ b/tools/cli-package/seal.mjs @@ -10,7 +10,7 @@ import { lstatSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; import { basename, isAbsolute, join } from 'node:path'; import { inspectPackageArchive } from './archive.mjs'; import { PACKAGE_TEST_MODE, originFromEnvironment, validatePackageReceipt } from './binding.mjs'; -import { flatFiles, oneTarball, readOrdinaryFile, sha256 } from './files.mjs'; +import { flatFiles, oneTarball, readOrdinaryFile, sha256, byCodeUnit } from './files.mjs'; import { MANIFEST_FILE, RECORD_BYTES, sealPrivateManifest, verifySealedDist } from './manifest.mjs'; import { symbolicateCandidate } from './symbolicate.mjs'; @@ -74,15 +74,15 @@ export function finalizedPublicFiles(root, staging) { actual.push({ path, bytes: bytes.length, sha256: sha256(bytes) }); } assert.deepEqual( - readdirSync(join(cli, 'dist')).sort(), - outputs.map((path) => basename(path)).sort(), + readdirSync(join(cli, 'dist')).sort(byCodeUnit), + outputs.map((path) => basename(path)).sort(byCodeUnit), 'PACKED_CLI_EXTRA_FINAL_OUTPUT', ); const record = JSON.parse(readOrdinaryFile(join(staging, 'finalized.json'), 1024 * 1024)); - const byPath = (left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0); + const byPath = (left, right) => byCodeUnit(left.path, right.path); assert.deepEqual( [...record.outputs].sort(byPath), - actual.sort(byPath), + [...actual].sort(byPath), 'PACKED_CLI_FINALIZED_RECORD_DRIFT', ); const canonical = flatFiles(join(root, 'packages/db/drizzle'), 'package/drizzle'); diff --git a/tools/cli-package/smoke/index.mjs b/tools/cli-package/smoke/index.mjs index e609981b..bc1f1f08 100644 --- a/tools/cli-package/smoke/index.mjs +++ b/tools/cli-package/smoke/index.mjs @@ -14,7 +14,9 @@ export const INSTALLED_SMOKE_ROOT_PREFIX = 'installed-smoke-'; * @param {string} supportRoot */ export function createInstalledSmokeRoot(supportRoot) { - return mkdtempSync(join(supportRoot, INSTALLED_SMOKE_ROOT_PREFIX)); + // `supportRoot` is the CI job's own absolute directory, already resolved and checked by the worker; + // no model supplies it, which is what Sonar's S8707 taint rule assumes of a command-line argument. + return mkdtempSync(join(supportRoot, INSTALLED_SMOKE_ROOT_PREFIX)); // NOSONAR — see above } /** diff --git a/tools/cli-package/source-map.mjs b/tools/cli-package/source-map.mjs index a27e35bd..214776ba 100644 --- a/tools/cli-package/source-map.mjs +++ b/tools/cli-package/source-map.mjs @@ -59,7 +59,7 @@ export function decodeMappings(mappings, { sources, names }) { const finish = () => { if (fields.length !== 1 && fields.length !== 4 && fields.length !== 5) refuse('MAPPINGS_FIELDS'); - const line = lines[lines.length - 1]; + const line = lines.at(-1); column += fields[0]; if (column < 0) refuse('MAPPINGS_RANGE'); if (line.length > 0 && column <= line[line.length - 1].generatedColumn) diff --git a/tools/cli-package/symbolicate.mjs b/tools/cli-package/symbolicate.mjs index e61fedbe..42f03f76 100644 --- a/tools/cli-package/symbolicate.mjs +++ b/tools/cli-package/symbolicate.mjs @@ -337,7 +337,7 @@ export async function symbolicateCandidate({ }; const reached = []; for (const frame of frames) { - if (frame.async || frame.file === null || !frame.file.startsWith('file:')) continue; + if (frame.async || !frame.file?.startsWith('file:')) continue; let path; try { path = realpathSync.native(fileURLToPath(frame.file)); diff --git a/tools/cli-package/ustar-fixture.mjs b/tools/cli-package/ustar-fixture.mjs index e48d8711..b72277e5 100644 --- a/tools/cli-package/ustar-fixture.mjs +++ b/tools/cli-package/ustar-fixture.mjs @@ -3,11 +3,11 @@ * can build a package tarball from an in-memory inventory. `package/dist/index.js` is 0755, every * other entry 0644. The dialect itself is `ustar.mjs`'s. */ -import { ustarEntry, writeChecksum, writeUstar } from './ustar.mjs'; +import { ustarEntry, writeUstar } from './ustar.mjs'; const packageMode = (path) => (path === 'package/dist/index.js' ? 0o755 : 0o644); -export { writeChecksum as checksum }; +export { writeChecksum as checksum } from './ustar.mjs'; /** @param {string} path @param {Uint8Array} bytes */ export function entry(path, bytes) { From 002ff6d776686a407f61fff9ef3f8a8bde1d15f6 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Wed, 4 Nov 2026 13:56:43 +0300 Subject: [PATCH 93/97] ci(repo): require the Windows job, scope the npm token, report skipped lanes Maintainer rulings on the PR #96 review (2026-11-04): - windows-concurrency drops its advisory label; it carries ADR-0106's Windows filesystem-policy scenarios, which name required CI. Branch protection adds it as a required check. - publish runs in the `release` deployment environment, which is to hold NPM_TOKEN and its required reviewer; a repository secret is readable by any workflow a writer pushes. The guard is unchanged, and release.yml and packaging.md record that D59's release ADR supersedes its condition and must override ADR-0111's "missing custody cannot downgrade to test mode". - The model-catalog nightly names each provider lane live or skipped, as a warning and in the job summary (only DEEPSEEK_API_KEY is configured). - The coverage job's allowance rises 30 -> 45 minutes (15-27 measured) and the secrets causal-control legs' 60 -> 75 (up to 48 measured); their pins follow. The baseline-predecessor fetch merges a nested if (S1066). Refs: ADR-0106, ADR-0111 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 40 ++++++++++++------------ .github/workflows/models-catalog.yml | 27 ++++++++++++++++ .github/workflows/release.yml | 9 ++++++ docs/reference/cli/packaging.md | 8 +++++ docs/reference/cli/regression-harness.md | 6 ++-- tools/ci/require-packed-cli.test.mjs | 3 ++ tools/ci/require-w8-acceptance.test.mjs | 8 ++--- tools/ci/workspace-scheduling.test.mjs | 2 +- 8 files changed, 76 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 89d4d55c..6e224e37 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,10 +3,12 @@ # Runs lint + typecheck + test across every workspace in dependency order with the # Turborepo cache, plus the formatting check and the no-vendor-type-across-the-seam fence. # -# Branch protection (set in the GitHub repo settings, not here): `ci` and `coverage` are the REQUIRED -# status checks to merge into `main`. Wire, usage, filesystem, secrets, packed CLI and product map control jobs feed `ci` through an explicit -# success-only aggregation; they cannot silently skip or fail while `ci` stays green. The other jobs -# self-label `(advisory)` — `floor-check`, `peer-dep-gate` and `windows-concurrency`. +# Branch protection (set in the GitHub repo settings, not here): `ci`, `coverage` and `windows-concurrency` +# are the REQUIRED status checks to merge into `main` (the Windows job since 2026-11-04: ADR-0106's +# acceptance names required CI for the platform-scoped claims it carries). Wire, usage, filesystem, secrets, +# packed CLI and product map control jobs feed `ci` through an explicit success-only aggregation; they cannot +# silently skip or fail while `ci` stays green. The other jobs self-label `(advisory)` — `floor-check` and +# `peer-dep-gate`. # # `coverage` enforces the testing.md >=90% line+branch floor for `@relavium/llm` and `@relavium/mcp` only. # `packages/core` is measured and printed by the same run but does not fail it: its branch margin is +0.83 @@ -114,11 +116,9 @@ jobs: shell: bash run: | BASELINE=$(node --input-type=module -e "import { baselineCommit } from './tools/budget-replay-compat/baseline-provenance.mjs'; process.stdout.write(baselineCommit)") - if ! git --no-replace-objects cat-file -e "$BASELINE^{commit}"; then - if ! git fetch --no-tags origin "$BASELINE"; then - echo "::error::Immutable replay predecessor $BASELINE is unavailable from origin; restore that exact object before running the offline gate." - exit 1 - fi + if ! git --no-replace-objects cat-file -e "$BASELINE^{commit}" && ! git fetch --no-tags origin "$BASELINE"; then + echo "::error::Immutable replay predecessor $BASELINE is unavailable from origin; restore that exact object before running the offline gate." + exit 1 fi git --no-replace-objects cat-file -e "$BASELINE^{commit}" @@ -299,9 +299,10 @@ jobs: # control runs its eight-MiB-heavy file in three phases (about 24 s locally). One export # of the whole cohort no longer fits the unchanged 64-MiB retained-evidence bound, so two # legs each run and export a disjoint round-robin half (`--shard k/2`) under the - # unchanged per-phase, case/hook, output, 2,000-file and 64-MiB bounds; the allowance - # matches the filesystem job's 60 minutes. This comment does not certify execution. - timeout-minutes: 60 + # unchanged per-phase, case/hook, output, 2,000-file and 64-MiB bounds. A leg took up to 48 minutes + # by 2026-11-04 (runner variance, not imbalance), so its allowance is 75 minutes rather than the + # filesystem job's 60. This comment does not certify execution. + timeout-minutes: 75 runs-on: ubuntu-latest strategy: fail-fast: false @@ -387,11 +388,9 @@ jobs: shell: bash run: | BASELINE=$(node --input-type=module -e "import { baselineCommit } from './tools/budget-replay-compat/baseline-provenance.mjs'; process.stdout.write(baselineCommit)") - if ! git --no-replace-objects cat-file -e "$BASELINE^{commit}"; then - if ! git fetch --no-tags origin "$BASELINE"; then - echo "::error::Immutable replay predecessor $BASELINE is unavailable from origin; restore that exact object before running the offline gate." - exit 1 - fi + if ! git --no-replace-objects cat-file -e "$BASELINE^{commit}" && ! git fetch --no-tags origin "$BASELINE"; then + echo "::error::Immutable replay predecessor $BASELINE is unavailable from origin; restore that exact object before running the offline gate." + exit 1 fi git --no-replace-objects cat-file -e "$BASELINE^{commit}" @@ -494,8 +493,9 @@ jobs: runs-on: ubuntu-latest # One file worker isolates unchanged admitted-size case budgets from instrumentation # contention. A measured 1,481-second complete run plus setup exhausted 25 minutes; - # reserve finite whole-suite headroom without changing case bounds or coverage thresholds. - timeout-minutes: 30 + # reserve finite whole-suite headroom without changing case bounds or coverage thresholds. By + # 2026-11-04 the job took 15-27 minutes, up to 91% of the former 30, so the allowance is 45. + timeout-minutes: 45 steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: @@ -542,7 +542,7 @@ jobs: # perm assertions are NOT exercised here (a documented Windows no-op — ADR-0050); nor is ink's raw-mode code, # which is TTY-gated (the driver-selection gate picks the plain driver without a TTY — see the smoke below). windows-concurrency: - name: windows · db concurrency + filesystem + headless smoke (advisory) + name: windows · db concurrency + filesystem + headless smoke runs-on: windows-latest timeout-minutes: 25 steps: diff --git a/.github/workflows/models-catalog.yml b/.github/workflows/models-catalog.yml index 44359506..7c637a93 100644 --- a/.github/workflows/models-catalog.yml +++ b/.github/workflows/models-catalog.yml @@ -113,6 +113,33 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} + # One configured key keeps the run from being a false green, but each missing key still skips its + # provider's lane while the job stays green (a PR #96 review: only DEEPSEEK_API_KEY was configured). + # Name every lane as live or skipped, as a warning and in the job summary, until all four keys exist + # (decision D37) and a missing one can fail the run instead. Only emptiness is tested; no value prints. + - name: Report which provider lanes run live + shell: bash + run: | + { + echo '### Live effort conformance lanes' + echo '' + echo '| Provider | Lane |' + echo '|---|---|' + } >> "$GITHUB_STEP_SUMMARY" + for provider in ANTHROPIC OPENAI GEMINI DEEPSEEK; do + variable="${provider}_API_KEY" + if [ -n "${!variable}" ]; then + echo "| ${provider} | live |" >> "$GITHUB_STEP_SUMMARY" + else + echo "::warning::${variable} is not configured; the ${provider} live effort probes are skipped, not verified." + echo "| ${provider} | skipped: no key |" >> "$GITHUB_STEP_SUMMARY" + fi + done + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} # The conformance suite (fixtures + the `it.skipIf( === '')` live cases). With the provider keys set here # the live cases execute — and a catalog that claims a tier a model rejects fails red — while a missing key # simply skips that provider's lane. Scoped to `src/conformance` so a flaky live case cannot widen the red diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 99de02f8..0e1d17a9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,6 +10,12 @@ # both exist every tag push stops at the guard, after the cross-OS smoke, so not even a v0.1.x patch publishes. # The flow description below is the pre-custody shape the guard now blocks. # +# Amended 2026-11-04 (decision D59, release path B): the condition above is to be superseded by a pre-1.0 +# release ADR that publishes the smoke-verified package-test tarball, with a sunset at 1.0 or when Steps +# 9c/9d land. That ADR must explicitly override ADR-0111's "missing custody cannot downgrade to test mode". +# Until it is Accepted and its guard change passes a security sitting, the guard below stays as it is. +# `publish` runs in the `release` deployment environment, so the npm token can be scoped to it. +# # Flow: `pack` builds the engine-inlined bundle and `pnpm pack`s it (resolving catalog:/workspace: to # concrete versions) into a tarball artifact → `smoke` installs THAT SAME tarball outside the workspace on ubuntu / # macOS / Windows and the minimum Node runtime and asserts the binary works (prebuilt native addons load, the bundled engine runs) → @@ -221,6 +227,9 @@ jobs: # manually dispatched run publish. Every manual run now stops after the cross-OS smoke. if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }} runs-on: ubuntu-latest + # The deployment environment that holds `NPM_TOKEN` and its required reviewer (repo settings); a + # repository-level secret is readable by any workflow a writer pushes. It sets no job environment. + environment: release timeout-minutes: 10 permissions: contents: read diff --git a/docs/reference/cli/packaging.md b/docs/reference/cli/packaging.md index 649e2eef..b991138c 100644 --- a/docs/reference/cli/packaging.md +++ b/docs/reference/cli/packaging.md @@ -344,6 +344,14 @@ recipient custody, the release state controller and the workflow graph are Steps Until those and Wave 4b's production recipient exist, the release workflow's publish job refuses every artifact (the [Interim release guard ruling](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) of 2026-10-19). +> Amended 2026-11-04: decision D59 (release path B, in the +> [MVP priority reset](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#mvp-priority-reset--2026-11-04)) +> replaces "until those exist" with a pre-1.0 release ADR that publishes the smoke-verified package-test +> tarball, with a sunset at 1.0 or when Steps 9c/9d land. That ADR must explicitly override +> [ADR-0111](../../decisions/0111-cli-meta-operations-use-a-lazy-packaged-runtime.md)'s "missing custody cannot +> downgrade to test mode". The guard stays in force until the ADR is Accepted and its guard change passes a +> security sitting. The publish job runs in the `release` deployment environment, which holds the npm token. + ### Private manifest and symbolication implementation boundary — Step 9a.2a Step 9a.2a (a [recorded split](../../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-step-9a2-preflight-decisions--2026-10-24) diff --git a/docs/reference/cli/regression-harness.md b/docs/reference/cli/regression-harness.md index ad914aff..3d056e02 100644 --- a/docs/reference/cli/regression-harness.md +++ b/docs/reference/cli/regression-harness.md @@ -127,12 +127,14 @@ journey, and the read-path query shapes. These are additive `.e2e.test.ts` suite ### The Windows lane (2.5.I S6) -An advisory `windows-concurrency` job in [`.github/workflows/ci.yml`](../../../.github/workflows/ci.yml) runs +A required `windows-concurrency` job in [`.github/workflows/ci.yml`](../../../.github/workflows/ci.yml) runs the full `@relavium/db` test suite (the concurrency/retry white-box tests among it) + the CLI concurrency/perf/session-chain harness on `windows-latest` (the WAL locking, the retry's `Atomics.wait` sleep, the two-process child spawn + `file://` import, and the native `better-sqlite3` addon are the parts most likely to diverge), plus a headless no-TTY smoke. It is a -**separate, advisory** job — the required check stays the ubuntu `ci` job. POSIX `0600`/`0700` permission +**separate** job and, since 2026-11-04, a **required** check beside the ubuntu `ci` job, because it carries +ADR-0106's Windows filesystem-policy scenarios ([ADR-0106](../../decisions/0106-media-publication-requires-bounded-verified-bytes.md) +names required CI for them). POSIX `0600`/`0700` permission assertions are a documented Windows no-op ([ADR-0050](../../decisions/0050-cli-history-db-at-rest-posture.md)) and are not exercised there; `release.yml` separately smokes the built binary across ubuntu/macOS/Windows. diff --git a/tools/ci/require-packed-cli.test.mjs b/tools/ci/require-packed-cli.test.mjs index ae651f84..3a143984 100644 --- a/tools/ci/require-packed-cli.test.mjs +++ b/tools/ci/require-packed-cli.test.mjs @@ -453,7 +453,10 @@ test('release publication is refused fail-closed until a verified release candid // runner's default shell and directory: no condition, shell, environment or defaults at the step, // job or workflow level, and no earlier `run` step that could write the job's environment. assert.deepEqual(Object.keys(guard).sort(), ['name', 'run']); + // The deployment environment scopes the npm token and its reviewer; it writes no job environment. + assert.equal(workflow.jobs.publish.environment, 'release'); assert.deepEqual(Object.keys(workflow.jobs.publish).sort(), [ + 'environment', 'if', 'name', 'needs', diff --git a/tools/ci/require-w8-acceptance.test.mjs b/tools/ci/require-w8-acceptance.test.mjs index 77c01ac9..670a4a4c 100644 --- a/tools/ci/require-w8-acceptance.test.mjs +++ b/tools/ci/require-w8-acceptance.test.mjs @@ -92,14 +92,14 @@ test('the actual required workflow joins both full offline controls and checks t assert.equal(secretsGuard['continue-on-error'], undefined); // Both families run as two legs that each run and export a disjoint half under the unchanged // 64-MiB retained-evidence bound, which one export of either whole cohort no longer fits. - for (const [kind, directory] of [ - ['fs', 'filesystem'], - ['secrets', 'secrets'], + for (const [kind, directory, minutes] of [ + ['fs', 'filesystem', 60], + ['secrets', 'secrets', 75], ]) { const job = workflow.jobs[`${kind}-mutations`]; assert.equal(job['continue-on-error'], undefined); assert.equal(job.if, undefined); - assert.equal(job['timeout-minutes'], 60); + assert.equal(job['timeout-minutes'], minutes); // Two legs, each running and exporting a disjoint half; neither may stop the other early. assert.deepEqual(job.strategy, { 'fail-fast': false, matrix: { shard: [1, 2] } }); assert.ok(job.steps.some((step) => step.run === 'pnpm install --frozen-lockfile')); diff --git a/tools/ci/workspace-scheduling.test.mjs b/tools/ci/workspace-scheduling.test.mjs index 071ab632..94ce27a2 100644 --- a/tools/ci/workspace-scheduling.test.mjs +++ b/tools/ci/workspace-scheduling.test.mjs @@ -186,7 +186,7 @@ test('required and local CI share all workspace tasks and uncached per-invocatio } const floor = workflow.jobs['floor-check']; assert.equal(floor['timeout-minutes'], 25); - assert.equal(workflow.jobs.coverage['timeout-minutes'], 30); + assert.equal(workflow.jobs.coverage['timeout-minutes'], 45); assert.equal( workflow.jobs.coverage.steps.find((step) => step.run === 'pnpm coverage:enforced')?.name, 'Engine coverage floor (>=90% line+branch — llm, mcp enforced; core measured)', From 964994c7fa7995cddd31f658af00cb5dd6bbeff8 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Wed, 4 Nov 2026 13:56:57 +0300 Subject: [PATCH 94/97] docs(repo): restore current.md's dropped records and settle the review's rulings The D66 reset kept current.md's live status and moved its 2026-07-26 plan verbatim, but a PR #96 review found its other unique records had left the docs (983 of 1,844 non-empty lines). They are moved verbatim now, links rewritten and headings lowered by one: the Wave 2/3 sections, the rows the reset edited in place, and current.md's, CLAUDE.md's and AGENTS.md's status blocks to Phase 2.5.5; the W7/W8 and Phase 2.6.5 records to Phase 2.6.5; Phase 2's status to Phase 2; "Not started yet" to Phase 2.6. Five lines remain by design (old metadata, two re-titled headings, one paraphrased duplicate whose phase copy is canonical); both move notes say so. Maintainer rulings (2026-11-04): - code-review.md states the single-maintainer review practice. - The LLM backstop's shape list moves to its reference home, docs/reference/shared-core/diagnostic-secret-backstop.md; error-handling.md keeps the rule. The module comment no longer claims an LF/TAB-split opaque Bearer value is joined, and the reference records that limit. - ADR-0105/0106's 2026-10-25 corrections take the dated "Amended" form. - 2.6.Q owns the stream usage-capture item; Step 9d owns a global encrypt-once record; the un-deferred item is "the CR-71 residual". Also: README and the local-dev runbook describe what works (CLI MVP first; a build-then-run CLI loop; desktop and extension not in the repo yet); the Wave 3 count reads 32; MVP-3 and MVP-8 split the Home strip; the branch rule matches AGENTS.md; and deferred-tasks.md records the review's verified lows and the Sonar dispositions. Co-Authored-By: Claude Opus 5.5 --- README.md | 9 +- ...invocation-grants-not-project-authority.md | 22 +- ...ication-requires-bounded-verified-bytes.md | 30 +- docs/reference/cli/commands.md | 2 +- docs/reference/shared-core/README.md | 1 + .../shared-core/diagnostic-secret-backstop.md | 117 ++++ docs/roadmap/current.md | 36 +- docs/roadmap/deferred-tasks.md | 78 ++- docs/roadmap/phases/phase-2-cli.md | 209 ++++++ .../phase-2.5.5-hardening-and-remediation.md | 617 +++++++++++++++++- .../phase-2.6-conversational-authoring.md | 33 +- ...hase-2.6.5-core-reliability-remediation.md | 600 +++++++++++++++++ docs/runbooks/local-dev-setup.md | 31 +- docs/standards/code-review.md | 12 +- docs/standards/error-handling.md | 107 +-- packages/llm/src/scrub-secrets.ts | 10 +- 16 files changed, 1734 insertions(+), 180 deletions(-) create mode 100644 docs/reference/shared-core/diagnostic-secret-backstop.md diff --git a/README.md b/README.md index c1ae9b40..77123328 100644 --- a/README.md +++ b/README.md @@ -158,10 +158,11 @@ Relavium is under active development. The **CLI is published as v0.1.1**; the pu agent-session entry point, workflow runtime, inbound MCP client, and CLI management surface are implemented. **Phase 2.6.5 — Core Reliability Remediation is complete with approved deferrals**, with W8 merged in PR #92. The joint Phase 2.5.5/2.6 security remediation -Wave 2 merged to `main` on 2026-10-15 as PR #94. The current -engineering wave is “Clear the ground” (CLI safety, shared guards and decomposition -of the hot files needed by later product work); its first eight steps merged to `main` -on 2026-10-18 as PR #95. +Wave 2 merged to `main` on 2026-10-15 as PR #94, and the first eight steps of Wave 3 +(“Clear the ground”) followed on 2026-10-18 as PR #95. Since 2026-10-26 the **CLI MVP +comes first**: Wave 3 is paused, not dropped, while the work a CLI user feels — visible +run results, a chat agent that can edit and run commands, readable replies, conversational +authoring, web, the Home hub and MCP management — leads to `relavium` v0.2.0. Status changes quickly, so this README intentionally stays high-level. The canonical source for the exact active wave, completed work, and open reliability obligations is diff --git a/docs/decisions/0105-filesystem-roots-are-invocation-grants-not-project-authority.md b/docs/decisions/0105-filesystem-roots-are-invocation-grants-not-project-authority.md index 6214e7e5..390d70de 100644 --- a/docs/decisions/0105-filesystem-roots-are-invocation-grants-not-project-authority.md +++ b/docs/decisions/0105-filesystem-roots-are-invocation-grants-not-project-authority.md @@ -326,14 +326,14 @@ future feature, deferral or claim of merge-commit execution. ## Exact native identity correction — 2026-10-25 -The root pins this decision requires compared device and inode as JavaScript numbers. On -Windows `ino` is a 64-bit NTFS file ID, and Linux overlayfs's `xino` feature sets high inode -bits; neither fits a Number's 53 bits, so a replaced root could compare equal (Step 9a.2a's -push CI observed it once on the Windows runner). The policy pins and the workflow catalog's -open check now read and compare exact bigints; the catalog comparison is Windows' only swap -guard, since `O_NOFOLLOW` is undefined there. The decision is unchanged: this restores the -identity the pins already claimed, and permanent causal controls substitute identities that -differ only beyond a Number's precision. ReFS's 64-bit file IDs are not guaranteed unique; -that limit is recorded in [deferred-tasks.md](../roadmap/deferred-tasks.md#native-file-identity-on-windows--approved-increment-2026-10-24). -The [NI checkpoint](../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-implementation-checkpoint--2026-10-25) -owns review and acceptance; this note claims neither. +> Amended 2026-10-25: The root pins this decision requires compared device and inode as JavaScript numbers. On +> Windows `ino` is a 64-bit NTFS file ID, and Linux overlayfs's `xino` feature sets high inode +> bits; neither fits a Number's 53 bits, so a replaced root could compare equal (Step 9a.2a's +> push CI observed it once on the Windows runner). The policy pins and the workflow catalog's +> open check now read and compare exact bigints; the catalog comparison is Windows' only swap +> guard, since `O_NOFOLLOW` is undefined there. The decision is unchanged: this restores the +> identity the pins already claimed, and permanent causal controls substitute identities that +> differ only beyond a Number's precision. ReFS's 64-bit file IDs are not guaranteed unique; +> that limit is recorded in [deferred-tasks.md](../roadmap/deferred-tasks.md#native-file-identity-on-windows--approved-increment-2026-10-24). +> The [NI checkpoint](../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-implementation-checkpoint--2026-10-25) +> owns review and acceptance; this note claims neither. diff --git a/docs/decisions/0106-media-publication-requires-bounded-verified-bytes.md b/docs/decisions/0106-media-publication-requires-bounded-verified-bytes.md index ae98fc6f..3fcbcb34 100644 --- a/docs/decisions/0106-media-publication-requires-bounded-verified-bytes.md +++ b/docs/decisions/0106-media-publication-requires-bounded-verified-bytes.md @@ -550,18 +550,18 @@ future feature, deferral or claim of merge-commit execution. ### Exact native identity correction — 2026-10-25 -The 2026-10-13 correction made the persisted claim token exact but left every live comparison -on numbers: the root pin, file snapshots and the owned-file check that guards deletion, the -read's descriptor stats, the publish handle, and the sweep's claim-directory, child, marker -and moved-blob checks; `nativeFile` read a bigint but compared it through `Number()`. Each now -compares exact bigint device and inode, each read from a single stat; the file and directory -checks share one projection, in which `nlink`, `size` and `mtimeMs` keep their number values -(`mtimeMs` by Node's own formula). Node fills bigint stats from a signed array, so a 64-bit file -ID with its top bit set read negative: `nativeFile`'s Number cross-check refused it, and the -claim token would have too, retaining such a temporary forever. The token now encodes the -unsigned value; -every other value keeps a byte-identical token, so persisted claims stay compatible. The -[physical storage contract](../reference/shared-core/built-in-tools.md#physical-media-storage-and-limits) -owns the rules. Permanent causal controls and the -[NI checkpoint](../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-implementation-checkpoint--2026-10-25) -own the proof and its gates. The cooperative root/path floor is unchanged. +> Amended 2026-10-25: The 2026-10-13 correction made the persisted claim token exact but left every live comparison +> on numbers: the root pin, file snapshots and the owned-file check that guards deletion, the +> read's descriptor stats, the publish handle, and the sweep's claim-directory, child, marker +> and moved-blob checks; `nativeFile` read a bigint but compared it through `Number()`. Each now +> compares exact bigint device and inode, each read from a single stat; the file and directory +> checks share one projection, in which `nlink`, `size` and `mtimeMs` keep their number values +> (`mtimeMs` by Node's own formula). Node fills bigint stats from a signed array, so a 64-bit file +> ID with its top bit set read negative: `nativeFile`'s Number cross-check refused it, and the +> claim token would have too, retaining such a temporary forever. The token now encodes the +> unsigned value; +> every other value keeps a byte-identical token, so persisted claims stay compatible. The +> [physical storage contract](../reference/shared-core/built-in-tools.md#physical-media-storage-and-limits) +> owns the rules. Permanent causal controls and the +> [NI checkpoint](../roadmap/phases/phase-2.5.5-hardening-and-remediation.md#wave-3-ni-implementation-checkpoint--2026-10-25) +> own the proof and its gates. The cooperative root/path floor is unchanged. diff --git a/docs/reference/cli/commands.md b/docs/reference/cli/commands.md index d392768c..eb289b54 100644 --- a/docs/reference/cli/commands.md +++ b/docs/reference/cli/commands.md @@ -232,7 +232,7 @@ Both are **surface-agnostic** git-native YAML operations — they read/write `.r `create` and `import` emit these notices only after their YAML file is written. A refused write emits no authored-content notice or success output. -The review visits at most 4,096 nodes through depth 64, checks strings of at most 16,384 UTF-16 code units and at most 262,144 checked code units in total, including property names. It reports only a count of strings matching the finite detector, the [diagnostic secret backstop](../../standards/error-handling.md#diagnostic-secret-backstop): prefixed OpenAI/Anthropic or Google keys, URL userinfo (any userinfo for HTTP(S)/WS(S)/FTP(S), colon-bearing userinfo for the listed connection-string schemes), exact `api-key`/`x-api-key`/`x-goog-api-key` headers, Bearer/Basic credentials and literal secret query names. It does not classify every credential format or distinguish arbitrary opaque credential fragments from following diagnostic prose. Over-budget strings or structure, repeated objects and unsupported object shapes produce a separate fixed incomplete-review notice. Object-key enumeration relies on the existing parser source ceiling; the node limit is not an independent bound on that enumeration. Neither notice includes values or arbitrary property names. +The review visits at most 4,096 nodes through depth 64, checks strings of at most 16,384 UTF-16 code units and at most 262,144 checked code units in total, including property names. It reports only a count of strings matching the finite detector, the [diagnostic secret backstop](../shared-core/diagnostic-secret-backstop.md): prefixed OpenAI/Anthropic or Google keys, URL userinfo (any userinfo for HTTP(S)/WS(S)/FTP(S), colon-bearing userinfo for the listed connection-string schemes), exact `api-key`/`x-api-key`/`x-goog-api-key` headers, Bearer/Basic credentials and literal secret query names. It does not classify every credential format or distinguish arbitrary opaque credential fragments from following diagnostic prose. Over-budget strings or structure, repeated objects and unsupported object shapes produce a separate fixed incomplete-review notice. Object-key enumeration relies on the existing parser source ceiling; the node limit is not an independent bound on that enumeration. Neither notice includes values or arbitrary property names. For this detector probe only, a recognised bare template reference without a path or filter is projected to an already masked span, provided its raw reference contains no recognised credential shape. This keeps `Bearer {{secrets.token}}`, a header such as `x-api-key: Bearer "{{secrets.token}}"` (quoted or not), equivalent query placeholders and a connection string whose password is a placeholder (`postgres://app:{{secrets.db_password}}@db`) quiet. An HTTP(S)/WS(S)/FTP(S) URL with a literal username beside a placeholder password still counts, because those schemes mask any userinfo. Adjacent literals, credential-shaped identifiers, paths, filter arguments and unknown or malformed references retain their evidence. This projection neither resolves references nor changes authored content. diff --git a/docs/reference/shared-core/README.md b/docs/reference/shared-core/README.md index 8a583b4b..0adeb4f8 100644 --- a/docs/reference/shared-core/README.md +++ b/docs/reference/shared-core/README.md @@ -20,5 +20,6 @@ Part of [reference/](../README.md). | [run-plan.md](run-plan.md) | The `RunPlan` the DAG builder (1.M) compiles from a validated workflow — engine vertices, topological order, dependency edges, per-type config. | | [built-in-tools.md](built-in-tools.md) | Built-in tools available to local agents (read_file, run_command, web_search, git, …). | | [tool-registry.md](tool-registry.md) | The engine-side `ToolRegistry` + dispatch and the host-injected `ToolHost` capability seam — policy/mechanism split, dispatch lifecycle, guardrail enforcement, result bounding, untrusted-data taint (1.T / [ADR-0037](../../decisions/0037-engine-tool-execution-boundary.md)). | +| [diagnostic-secret-backstop.md](diagnostic-secret-backstop.md) | The finite secret shapes the LLM diagnostic backstop (`scrubSecrets`) masks in provider errors and CLI diagnostics, and the limits of that list. | | [mcp-integration.md](mcp-integration.md) | MCP: agents as MCP servers, and agents consuming MCP tools. | | [action-guard-seam.md](action-guard-seam.md) | The optional, host-injected `ActionGuard` seam an external action-governance control plane plugs into at the side-effecting tool boundary: decide/commit/compensate, verdict union, off-by-default ([ADR-0041](../../decisions/0041-external-action-governance-seam.md), Accepted — contract settled, implementation deferred to the Phase-2 enterprise track). | diff --git a/docs/reference/shared-core/diagnostic-secret-backstop.md b/docs/reference/shared-core/diagnostic-secret-backstop.md new file mode 100644 index 00000000..d4f40b31 --- /dev/null +++ b/docs/reference/shared-core/diagnostic-secret-backstop.md @@ -0,0 +1,117 @@ +# Diagnostic secret backstop — the finite shape list + +- **Status**: Accepted — the behavior `packages/llm/src/scrub-secrets.ts` implements +- **Canonical home**: the finite secret shapes `scrubSecrets` masks in provider diagnostics (`makeLlmError`'s `message` and `code`) and in the CLI's display sanitizer, and the documented limits of that list +- **Related**: [error-handling.md](../../standards/error-handling.md#diagnostic-secret-backstop) (the rule: a defense-in-depth backstop behind an adapter's own redaction), [built-in-tools.md](built-in-tools.md#finite-diagnostic-credential-masking) (core's journal and display projection, which differs from this list by design), [llm-provider-seam.md](llm-provider-seam.md) (the error model `makeLlmError` belongs to), [../../standards/security-review.md](../../standards/security-review.md) + +Moved here from [error-handling.md](../../standards/error-handling.md#diagnostic-secret-backstop) on +2026-11-04, unchanged: a PR #96 review found an exact specification in a standard, against the rule that one +lives only in its `docs/reference/` home. The standard keeps the rule and links here. + +- **Shapes.** OpenAI/Anthropic `sk-…` and Google `AIza…` key prefixes; a finite list of literal + secret query names, after `?`, `&` or `#` (an OAuth implicit-flow `#access_token=`); the exact + `api-key`, `x-api-key` and `x-goog-api-key` header names, also in quoted, JSON-encoded and + JSON-in-JSON (`\"x-api-key\":\"…\"`) objects; and `Bearer`/`Basic` authentication values. A header + value may begin with up to eight (possibly quoted, escaped, punctuation-glued or LF/TAB-split) + `Bearer`/`Basic` labels — a doubled `Bearer Bearer` is a common misconfiguration. Its token may + begin with any run of (escaped) quotes and ends at a delimiter, at escapes before one, or before a + nested header name, which then masks its own value. A label may be closed by its own quote before + its token (`"Bearer" `), and a run of up to eight concatenated labels (`BearerBearer`) + counts as one label. No label takes as its token another label that whitespace or the end of the + text follows, even one glued behind punctuation, quoted or concatenated, so a standalone value + with any number of separately stacked labels masks the token after the last one, and a label glued + to a token behind a non-word character masks its own token; a second label that other punctuation + follows (`Bearer Bearer;`) is masked as a token, which only over-masks. +- **Masking.** Every rule, the literal query names included, finds its matches in the original + text, and the masked ranges of all rules are merged before anything is replaced, so no rule + can erase the context another rule needs. A header, label or query rule masks only its token: + the header name, the separator, any labels and the query name stay visible, an LF/TAB-split + label keeps its spelling, and quotes around a token stay outside it. Masking a token that is + already exactly `[REDACTED]` changes nothing, so a masked token is a fixed point, quoted, + escaped or after labels. After a `Bearer`/`Basic` label, text glued to the marker + (`Bearer [REDACTED];tail`) is masked with it; a header token ends at `,`, `;` or `}`. One + pass can still glue a marker to the text after another rule's token, so the scrub repeats + until its output stops changing, at most three passes, and blanks the whole diagnostic if it + has not settled: a second scrub changes nothing. +- **URL userinfo.** The `?`/`#` rule decides where userinfo ends: a raw `?` or `#` before the + authority `@` begins public query or fragment text only when the text before it is host-shaped (a + dotted name or IPv4, `localhost` or a bracketed IP literal, with an optional port, at most 260 + characters in all) **and** an `=` assignment follows it, as in + `https://host.example?contact=a@b.example`. Masked userinfo that holds such a raw `?` or `#` + leaves the authority ambiguous: unless a host-shaped name follows its last `@`, the text there may + be the rest of a query value (`https://svc?password=P@ss`), so the mask runs through the + authority's end, the next `/`, whitespace, quote, `,` or `}`. The scheme rule then decides what is + masked: `http(s)`, `ws(s)` and `ftp(s)` mask any userinfo, while the connection-string schemes + `postgres`, `postgresql`, `mysql`, `mariadb`, `redis`, `rediss`, `mongodb`, `mongodb+srv`, `amqp`, + `amqps`, `ssh`, `sftp`, `ldap`, `ldaps`, `smtp` and `smtps` mask only colon-bearing userinfo + (`user:pass@`, `:pass@`), so a bare user such as `ssh://git@host` (or a colonless + `redis://#TOKEN@host`) stays visible. Userinfo whose password is exactly masked stays as it is: + beside any username under a connection-string scheme, or beside a masked or empty username under + any scheme; an empty password, or text glued after a masked one, is still masked. The + authored-content detector's projection of `postgres://app:{{secrets.password}}@db` to + `postgres://app:[REDACTED]@db` is therefore a fixed point. Core's journal projection applies the + same `?`/`#` predicate and also merges its rules' ranges, but masks a userinfo together with the + values its key and opaque-query rules recognize in it, its username's excepted, each to its own + end, and ends a value no rule recognizes at the last `@`, whose text after it stays as the host, + rather than masking through the authority's end + ([built-in tools](built-in-tools.md#finite-diagnostic-credential-masking)). +- **Split and normalized forms.** Every rule except the literal query names also probes the + terminal-normalized text; a credential revealed only by normalization masks the **whole** + diagnostic. The key prefixes, URL userinfo and the exact header names are additionally probed in + the LF/TAB-joined view at original-text boundaries — of the raw text too when terminal + normalization changed it, since a terminal sequence there may take a split credential's first + letter (ESC `A` before `AIza…`) — and a credential revealed only by such a split masks the whole + diagnostic too. A `Bearer`/`Basic` label may itself be split by LF/TAB, but its opaque value is not + joined: a value split by an LF or TAB is masked up to the separator and its tail stays visible + (documented 2026-11-04, after a PR #96 review found the module comment claiming otherwise). +- **Documented trade-offs, all failing closed.** A scheme glued to a word character is masked in + place; an unlisted scheme ending in a listed one (`gftp://`, `jdbc:mysql://`) matches by its + suffix; prose that the whitespace-stripped residual probe glues to a path-less URL and a later `@` + blanks the whole diagnostic, and so does a key prefix (`sk-`, `AIza`) that the same probe joins + with the following words to its minimum length, at the start of the text or, since the scrub + repeats, right after another rule's mask (`Authorization: Bearer xyz sk-ant api key invalid`); a + path-less terminal hyperlink target (OSC 8) that the raw LF/TAB-joined probe joins with a later + `@` (a hyperlinked table cell before an e-mail address) blanks it as well; a host fragment with no + assignment (`https://host.example#sec@x`), a single-label host query and an over-long host-shaped + prefix are masked, and so is the text after the last `@` of such an ambiguous authority unless it + is host-shaped; a `Bearer`/`Basic` label that ends a line takes the next token with it; an opaque + value after a label ends only at whitespace, so in compact JSON it masks to the end of the run; + nine or more concatenated labels, read in the raw text as well as the normalized one, and a token + that begins with `,`, `;` or `}` after a `Bearer`/`Basic` label in a header value, blank the whole + diagnostic. Past eight separate labels after a header name, and in a value of labels alone glued + behind punctuation (`Bearer |Bearer`), the outcome depends on what precedes the header name: when + its nearest non-whitespace character is a word character, `-` or `_`, the diagnostic stays in + place (the last label masks any token); otherwise, the start of the text included, it is blanked. + No token is visible either way, and the output is stable under a second scrub. +- **Limits.** Userinfo under an unlisted scheme is not masked, including driver-qualified spellings + such as `postgresql+psycopg2://`. Opaque `Bearer`/`Basic` values, `api-key` header values and + literal secret query values end at any whitespace, LF and TAB included, so a fragment after such a + break stays visible, as it does after an ordinary space; a label glued to a quoted token without + whitespace (`Bearer""`) is likewise outside the shape, as is a token after a JSON escape + (`\n`, `\t`) that stands for the whitespace after a label. A JSON escape right after a header + name's colon starts its token, so a value glued to the escape is masked with the escape's letter + (`"x-api-key":"\n"` → `"x-api-key":"\[REDACTED]"`), while a quoted value behind such an escape + stays visible (`x-api-key:\t""`). A terminal sequence with a parameter whose final byte is a + shape's first letter (ESC `[` `1` `B` before `earer `), and a terminated device-control, + start-of-string or application-program string (ESC `P`, ESC `X` or ESC `_`, ended by ST or BEL) + glued before a shape, hide that shape from every view: in the raw text the sequence's last byte + glues to the shape's first letter, and normalization removes it. A word glued before `sftp://` on + the same line (`statusftp://user@h`) reads as `sftp`, so its colonless userinfo stays visible like + any `sftp` user: every whitespace-joined probe, the CLI's display residual among them, glues prose + to a scheme the same way, and reading it as `ftp` would blank an ordinary + `connect to sftp://user@host` diagnostic. (A word that ends a line before `ftp://` is no such + glue: an LF/TAB-split ftp credential after it blanks the diagnostic.) Malformed URLs + (`https:/u:p@h`), a raw `/` inside a password, invisible characters inside a scheme and JSON whose + quotes are themselves escaped as `\u0022` are outside the finite shapes. In a JSON-encoded + diagnostic, an escape written as two characters (`\n`, `\t`, `\r`) and glued before a key prefix, + a header name or a label defeats that shape's word boundary, so the credential after it stays + visible (`"key rejected:\nsk-…"`). Core's journal projection shares this limit for key prefixes + and scheme labels, while core's summary also reads each such escape as its separator and masks the + credential after it; core's key/value rule's word-character prefix still reads a header or + secret-key name glued after such an escape (`"err:\napi-key: …"`) and masks the value with it. A header value + that begins with `,`, `;` or `}` and has no label before it is an empty token, so the text after + it stays visible. The backstop has no generic key/value rule: a secret key inside a URL authority + whose value holds `@` — after `;` (`https://svc;a=1;password=P@ss…`), after `&` unless it is one + of the literal secret query names, or on the next line of a JSON-encoded file whose break is the + escape `\n` — leaves its value's text after the last `@` visible as the host. Core's key/value + rule reads that key and masks its value. diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index 81de18f7..083e9fd4 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -82,7 +82,7 @@ Wave 3 split as `row ` in the [increment table](phases/phase-2.5.5-hardening- drafts for the new ADRs: the pre-1.0 release path (D59), the toolbelt plus tool-render/approval-preview contract (phase 2.6 ADR row 9, split so `ask_user` comes later), the in-house markdown architecture (ADR row 11, part 1; no dependency), the ADR-0057 amendments (default grant and `default_mode`, D62; denial feedback, D67), the - keymap (D68), the `run_command` approval card (D63) and the ADR-0095 amendment for CR-71 (D67). Maintainer + keymap (D68), the `run_command` approval card (D63) and the ADR-0095 amendment for the CR-71 residual (D67). Maintainer gates: CI provider keys for the live-nightly lane (D37), provider spend for a manual three-provider turn and a Windows machine for MVP-10. A stale-entry sweep of [deferred-tasks.md](deferred-tasks.md) and the phase 2.6 bullets that 2.6.5 and Wave 2–3 work already satisfied. @@ -104,12 +104,14 @@ Wave 3 split as `row ` in the [increment table](phases/phase-2.5.5-hardening- ranges and the spill store (MVP-A9); 2.6.M · `write_file`'s approval diff (#305) and **Tool-call rendering v2** (sanitized target kept after the turn; assistant text between tool calls kept); 2.6.M · **Default chat agent grant review** as amended by D62 (MVP-A14) with `[chat].default_mode` (MVP-A13); the 80×24 guard on - both chat surfaces and the Home strip row budget (#63, row 15c split). + both chat surfaces and the Home strip row budget (#63, row 15c split) — MVP-3 owns the strip's geometry + and its no-hidden-input rule, MVP-8 its content. 4. **MVP-4 — The agent runs commands and keeps its footing.** 2.6.G · **Permission management** as ruled by D63 (MVP-A15; `!` uses the same card); the default agent gains `run_command`; the keymap, including Ctrl-C at a pending approval rejecting it (MVP-A12; row 11b split); denials returned to the model (MVP-A10); a failed, - denied or aborted turn kept in context as a content-free interruption marker (CR-71, un-deferred, D67); - configurable, finite turn ceilings with conservative defaults (MVP-A11). + denied or aborted turn kept in context as a content-free interruption marker (the CR-71 residual, + un-deferred, D67; CR-71 itself closed in W7); configurable, finite turn ceilings with conservative + defaults (MVP-A11). 5. **MVP-5 — Context and parity (D71).** Anthropic prompt caching (2.5.5.B · #172); a project instructions file and environment block (MVP-A16); 2.6.E · **Type-ahead message queue**; `chat --model/--provider` and a one-shot prompt mode (MVP-A17). @@ -120,8 +122,8 @@ Wave 3 split as `row ` in the [increment table](phases/phase-2.5.5-hardening- 7. **MVP-7 — Web.** G30 (rows 13a–13c, pulled forward; 13c is re-scoped when it starts because Step 12b is paused); 2.6.M · **`web_search` activation**; web fetch through `http_request` under the chat authority. 8. **MVP-8 — The Home as hub.** Phase 2.6 ADR row 6; 2.6.H's history data layer as far as G needs it; 2.6.K's - shared resume core; 2.6.G's `/agents`, the actionable strip, `/workflows` (defined, history, run detail) and the - bound-agent view; Home `/help`; `logs --follow/--failed`. + shared resume core; 2.6.G's `/agents`, the actionable strip's content (within MVP-3's row budget), + `/workflows` (defined, history, run detail) and the bound-agent view; Home `/help`; `logs --follow/--failed`. 9. **MVP-9 — MCP and provider management.** Phase 2.6 ADR row 7; MCP servers attached to the default agent under ADR-0084 consent (MVP-A21); 2.6.I · **`/providers`**, `/mcp`, **`relavium mcp list/add/remove/test`** and MCP header auth; the masked `provider set-key` prompt (MVP-A20). @@ -145,7 +147,10 @@ stays so (D60); the published `0.1.1` still prints unsanitized model output in ` > ` · ()`. The waves below keep the order derived on > 2026-07-26; since 2026-11-04 they run **after** the MVP above. The original plan's baseline, axes, Waves 0–1 > and the PR #81 closing list (the only definition of `#W15-1`…`#W15-24`, which code comments cite) are preserved -> verbatim in the [Phase 2.5.5 register](phases/phase-2.5.5-hardening-and-remediation.md#execution-record-moved-from-currentmd--2026-11-04). +> verbatim in the [Phase 2.5.5 register](phases/phase-2.5.5-hardening-and-remediation.md#execution-record-moved-from-currentmd--2026-11-04), +> and so, since the [PR #96 review correction](phases/phase-2.5.5-hardening-and-remediation.md#moved-by-the-pr-96-review-correction--2026-11-04), +> are this page's other records: the Wave 2 and Wave 3 sections, its status blocks, and the W7, W8 and Phase 2.6.5 +> records in [Phase 2.6.5's](phases/phase-2.6.5-core-reliability-remediation.md#execution-record-moved-from-currentmd--2026-11-04). ```mermaid flowchart TD @@ -359,7 +364,7 @@ unanswered**; the remainder sit inline in their own phase-file bullet. | D64 | MVP | Acceptance bar for MVP increments | ✅ **Ruled 2026-11-04 — tiered** (see [Next work](#next-work--the-cli-mvp)) | | D65 | MVP | Paused Wave 3 increments' user-visible pieces | ✅ **Ruled 2026-11-04:** six recorded splits (10b, 15a, 15c, 11b, 16b, 17a) and #61 move into the MVP; Steps 12/14 stay paused | | D66 | Gate 0 | `current.md`'s history | ✅ **Ruled 2026-11-04:** relocate the unique records, delete the duplicated history, keep every inbound anchor | -| D67 | MVP-4 | Context fidelity and "a user deny is final" | ✅ **Ruled 2026-11-04:** CR-71 un-deferred into MVP-4 (content-free marker), denials returned to the model, finite configurable ceilings; CR-70 with ADR-0096 §5 in v0.2.1 | +| D67 | MVP-4 | Context fidelity and "a user deny is final" | ✅ **Ruled 2026-11-04:** the CR-71 residual un-deferred into MVP-4 (content-free marker; CR-71 itself closed in W7), denials returned to the model, finite configurable ceilings; CR-70 with ADR-0096 §5 in v0.2.1 | | D68 | MVP-1/2/4 | First-run defaults and the keymap | ✅ **Ruled 2026-11-04:** env-var key fallback, the interactive output budget, the default model following the stored key, a minimal `init`, and one keymap ADR (Ctrl+C interrupts the turn, a second press ends the session, Ctrl+C at a pending approval rejects it first) superseding the 2026-10-19 `/cancel` wording | | D69 | MVP | Order of the added lanes | ✅ **Ruled 2026-11-04:** authoring → web → Home → MCP | | D70 | MVP | npm pre-release before v0.2.0 | ✅ **Ruled 2026-11-04 — none** | @@ -390,15 +395,18 @@ places, each deliberate: ## Standing rules -> **Branch from `origin/development`, never from a local `main`.** A long-lived `development` merged into -> `main` by merge commit leaves a stale local `main` behind after every PR, silently and without conflict. -> Run `git fetch` and cut from `origin/development`. +> **Work on `development` itself** ([AGENTS.md](../../AGENTS.md)); create another branch only when the +> maintainer asks, and then cut it from `origin/development`, never from a local `main`. A long-lived +> `development` merged into `main` by merge commit leaves a stale local `main` behind after every PR, silently +> and without conflict. Run `git fetch` first. - The gate is `pnpm run ci` plus `pnpm coverage:enforced`; `development` is pushed after each accepted increment and its push CI is observed. -- `main` carries branch protection (PR required, `lint · typecheck · test` and the coverage floor required, - force-push and deletion blocked) and a `v*` tag ruleset limited to the maintain/admin roles; `release.yml` - asserts its ref is reachable from `main`. +- `main` carries branch protection (PR required; `lint · typecheck · test`, the coverage floor and, since + 2026-11-04, the Windows job required; no approval count, since GitHub refuses a self-approval — see + [code-review.md](../standards/code-review.md); force-push and deletion blocked) and a `v*` tag ruleset + limited to the maintain/admin roles; `release.yml` asserts its ref is reachable from `main` and publishes + from the `release` deployment environment. ## History diff --git a/docs/roadmap/deferred-tasks.md b/docs/roadmap/deferred-tasks.md index e2123883..e74c90cc 100644 --- a/docs/roadmap/deferred-tasks.md +++ b/docs/roadmap/deferred-tasks.md @@ -963,6 +963,82 @@ to the file it names. tools/cli-package/release-recipient.test.mjs, tools/cli-package/release-binding.test.mjs, tools/cli-package/release-custody.test.mjs)* +## PR #96 review — recorded findings and Sonar dispositions (2026-11-04) + +Recorded by the PR #96 review correction. The review's verified high and medium findings and SonarCloud's 27 bugs and +10 vulnerabilities were fixed in that correction; these are the verified lows and the code smells the maintainer +ruled on 2026-11-04 to record rather than refactor now. **Trigger for every entry:** the next change to the file it +names, unless the entry names another. + +**Sonar code smells (gate-neutral; maintainability stays A):** + +- [ ] **Cognitive complexity (`S3776`) in fifteen functions.** *Low.* `packages/core/src/tools/bounding.ts` (five, + 25–38), `packages/llm/src/scrub-secrets.ts:411`, `tools/cli-package/source-map.mjs:47` (35), `map-controls-core.mjs`, + `release-custody.mjs`, `candidate.mjs`, `release-binding.mjs`, `symbolicate.mjs`, `ustar.mjs`, + `tools/bundle-closure/closure.mjs` and `inventory.mjs`. Each is security-critical redaction or release tooling, so a + refactor needs its own mutation sweep and review. *(low · the files named)* +- [ ] **Regex complexity (`S5843`) in eight patterns.** *Low.* `bounding.ts`'s standalone and key rules and five + `scrub-secrets.ts` rules: their "complexity" is the length of a finite name or shape alternation (as the existing + `NOSONAR` notes say), and each is a causal-control anchor. The three `S5869` reports on `scrub-secrets.ts:138` are + the `i` flag making `A-Z` and `a-z` overlap, not a duplicate. **Re-check:** split a rule only with its controls. + *(low · packages/core/src/tools/bounding.ts, packages/llm/src/scrub-secrets.ts)* +- [ ] **`S1994` on the closing-view loop.** *Low.* `bounding.ts`'s backward scan tests `next` but steps `index`; a + `while` rewrite must keep the step that its `continue` skips. *(low · packages/core/src/tools/bounding.ts)* +- [ ] **`S6582` in `media-store-paths.ts` (three).** *Low.* The optional-chain rewrites are equivalent, but the lines + are filesystem causal-control anchors; change them with those controls. *(low · packages/db/src/media-store-paths.ts)* +- **DECLINED — `S9382` (seven sequential awaits) and `S7758` (five `charCodeAt`).** The awaits are ordered process + steps by design (`map-controls-core.mjs`, `pipeline.mjs`, `symbolicate.mjs`), and `source-map.mjs` is a code-unit + VLQ parser that advances one code unit at a time, which `codePointAt` would misstate. + +**Redaction:** + +- [ ] **The LLM backstop leaves a `/` in a URL password.** *Low.* `scrubSecrets('https://user:pa/ss@host')` keeps the + password, because its userinfo pattern stops at `/`; core masks it since this correction. Matching core needs its + port guard (`host:8080/a@b` stays public). *(low · packages/llm/src/scrub-secrets.ts)* +- [ ] **A glued `-pSECRET` (`mysql -uroot -pSecret`) is not masked.** *Low.* A glued `-p` is ambiguous with + `cp -pR` and `tar -pxf`, so only the separate and `=` spellings are read. *(low · packages/core/src/tools/bounding.ts)* +- [ ] **The object-key rule masks count keys such as `max_tokens`.** *Low.* Its plural folding reads `tokens` as + `token`, so two calls that differ only in `max_tokens` share a journal identity. The argv rule reads words as written + and does not. *(low · packages/core/src/tools/bounding.ts)* +- [ ] **Zero-width characters break the `sk-` and `Bearer` shapes; header names are matched with hyphens only.** + *Low; verified.* *(low · packages/llm/src/scrub-secrets.ts)* +- [ ] **A model-supplied tool name reaches an error message unmasked and unbounded.** *Low; verified.* *(low · + packages/core/src/tools/errors.ts:21)* +- [ ] **Two redaction engines and two fixed-point copies; the parity test compares booleans only.** *Low; verified.* + *(low · packages/core/src/tools/bounding.ts, packages/llm/src/scrub-secrets.ts)* +- [ ] **Permanent causal controls for this correction's three mechanisms.** *Low.* The credential-flag words, the + `,`/`/` password reading and the omitted-media bytes are pinned by permanent tests and a manual mutation sweep (every + mutant killed), not by `check:secrets-mutations` controls. The secrets evidence has about 15 MiB of headroom per + shard. *(low · tools/secrets-controls/mutations.mjs)* + +**Release, custody and tooling (verified lows):** + +- [ ] **`isProductionRecipient` accepts seam values; the distinction rests on a grep control.** *(low · + tools/cli-package/release-recipient.mjs:52)* Trigger: Step 9d or Wave 4b. +- [ ] **A revocation is visible only in the committed recipient snapshot.** *(low · docs/reference/cli/packaging.md)* +- [ ] **`resolveGpg` checks `isFile`, not `X_OK`.** *(low · tools/cli-package/gnupg.mjs:33)* +- [ ] **A developer's own `core.excludesFile` directory can let private untracked files into the candidate.** The + public tarball is not affected; an explicit `RELAVIUM_GIT_EXCLUDES_FILE` naming a directory already fails in Git. + *(low · tools/cli-package/candidate.mjs)* +- [ ] **`prepack.mjs` selects the finalizer by environment variable; `symbolicate.mjs` may load TypeScript from a parent + directory.** The closure check is equivalent for the first. *(low · tools/cli-package/prepack.mjs:36, + tools/cli-package/symbolicate.mjs:306)* +- [ ] **`runCommand` does not follow detached grandchildren after a successful exit; `causal-controls` starts a + detached group without a Ctrl-C handler; the smoke's cleanup can skip its PID kill after a parse error.** *(low · + tools/cli-package/process.mjs, tools/causal-controls/, tools/cli-package/smoke/)* +- [ ] **`mcp-orphan.e2e` scans the process table by fixture name, which sibling tests share; `vi.waitFor` keeps its + 1-second default.** *(low · packages/mcp)* +- [ ] **The case-insensitive custody tests run in no CI job.** *Medium as reviewed, low in practice.* Ubuntu skips + them and no macOS or Windows job runs `custody.test.mjs`; the local `pnpm run ci` on macOS runs them. A CI lane + needs a macOS job inside `ci`'s success aggregation. *(low · tools/cli-package/custody.test.mjs:830, .github/workflows/ci.yml)* +- **DECLINED — declaring `RELAVIUM_CLI_CANDIDATE_STAGING` in `turbo.json`.** The candidate build never runs through + Turbo (`pnpm exec tsc` and `pnpm pack`), and Turbo 2's strict environment mode withholds an undeclared variable, so + a Turbo build is always map-free; declaring it would let one become a candidate build. + +**Not yet recorded here:** the review's unverified lows (its section F) wait for the consolidated file from that +review session, and two roadmap lows ("DONE entries whose body still describes the open state", "MVP items without an +annotation") need the entries they name. + ## Wave 3 NI — recorded low findings (2026-10-25) Round 1 of the native-identity increment at `fc5af701` accepted it with these lows @@ -1094,7 +1170,7 @@ surface ships. CR-95's long continuation and the exact-capacity residual remain - **Trigger.** The first surface that needs to show, or resume from, what a failed turn did — a coding assistant replaying a half-finished edit, or an audit view over a session's tool use. *(medium · apps/cli/src/chat/persister.ts, packages/core/src/engine/agent-session.ts; `CR-71`, ADR-0095)* - **Pulled into the CLI MVP 2026-11-04 (D67, MVP-4):** a failed, denied or aborted turn stays in context as a + **Pulled into the CLI MVP 2026-11-04 (D67, MVP-4) as "the CR-71 residual"** — CR-71 itself closed in `W7`: a failed, denied or aborted turn stays in context as a content-free interruption marker (ADR-0095 §1), behind an ADR-0095 amendment drafted in Gate 0. - [ ] **Two at-rest residuals `secure_delete` does not cover: a blocked WAL checkpoint, and pages freed before the diff --git a/docs/roadmap/phases/phase-2-cli.md b/docs/roadmap/phases/phase-2-cli.md index 91a92851..1121f33c 100644 --- a/docs/roadmap/phases/phase-2-cli.md +++ b/docs/roadmap/phases/phase-2-cli.md @@ -911,3 +911,212 @@ All must be true to start Phase 3 (desktop): | No OS keychain in CI/headless hosts. | Cannot run in pipelines. | Documented env-var → `secrets.enc` fallback with a hard error on absence, no silent plaintext (2.C). | | High token volume overwhelms the TUI. | Dropped events / janky render. | Throttle/batch with `sequenceNumber` gap-detection + resync; verify event count vs JSON stream (2.E). | | Cross-OS install differences. | "Works on my machine" binary. | Multi-OS smoke tests of the published artifact before sign-off (2.L). | +## Status record moved from current.md — 2026-11-04 + +> Moved verbatim from `docs/roadmap/current.md` as it stood at `5a328bca` by the +> [PR #96 review correction](phase-2.5.5-hardening-and-remediation.md#moved-by-the-pr-96-review-correction--2026-11-04). +> Only relative links were rewritten for this directory. + +**Phase 2 — CLI (milestone M3) is feature-complete** (every in-phase workstream 2.A–2.S merged; published as +**v0.1.1**). The CLI is the first real +`@relavium/core` consumer and doubles as the engine's regression harness — validating the +engine API ergonomics before the desktop and VS Code surfaces. **Landed:** the CLI skeleton + +process contract (**2.A**) and the two-level config-resolution loader (**2.B**), both ✅ Done +(PR #40, 2026-06-22) behind [ADR-0047](../../decisions/0047-cli-framework-commander-ink-clack.md) +(commander/ink/@clack) and [ADR-0048](../../decisions/0048-toml-config-parser.md) (smol-toml); and +**2.D** (`relavium run` wired to `@relavium/core` — the M3 keystone and first real engine +consumer), ✅ Done (PR #41, 2026-06-22), which also adds the `defaultProviders()` seam registry; +and **2.F** (the `--json` CI machine-output contract — pure-NDJSON stdout, diagnostics → stderr), +✅ Done (PR #42, 2026-06-22) behind [ADR-0049](../../decisions/0049-cli-machine-output-contract.md); +and **2.K** (the engine regression harness, now the engine's CI regression gate), ✅ Done +(PR #43, 2026-06-23) — **reaching global milestone M3**; and **2.H** (durable local run history via +`@relavium/db` — the `RunStore` writer + read API the gate-resume/list/logs/status surfaces consume), +✅ Done (PR #44, 2026-06-23) behind [ADR-0050](../../decisions/0050-cli-history-db-at-rest-posture.md); +and **2.C** (the `relavium provider` commands — a provider registry + API keys in the OS keychain via +`@napi-rs/keyring`, resolved keychain → `RELAVIUM__API_KEY` env var → error), ✅ Done +(PR #45, 2026-06-23) behind [ADR-0019](../../decisions/0019-cli-node-keychain-library.md) + +[ADR-0006](../../decisions/0006-os-keychain-for-api-keys.md) (no new ADR — `secrets.enc` deferred past v1.0); +and **2.E** (the `ink` streaming TUI — the third `RunRenderer` over the one event bus: live per-node status ++ spinners, the active node's streaming tokens, a running cost footer, a persistent final summary; cooperative +Ctrl-C cancel), ✅ Done (PR #46, 2026-06-24) behind [ADR-0047](../../decisions/0047-cli-framework-commander-ink-clack.md) +(ink + React 19, confined to `apps/cli`; no new ADR); +and **2.G** (the interactive human-gate prompt — a `@clack/prompts` card during `run` — plus the out-of-band +`relavium gate ` cross-process resume: reload snapshot → reconstruct checkpoint → `resumeFromCheckpoint`, +idempotent, secret-input fail-closed), ✅ Done (PR #47, 2026-06-24) behind +[ADR-0047](../../decisions/0047-cli-framework-commander-ink-clack.md) (`@clack/prompts`; Node floor 20.11→20.12; +no new ADR) — **fully closing 2.K's deferred gate-resume half**; +and **2.I** (the read commands `list` / `logs` / `status` / `gate list` over durable history — go/no-go #2, the +read side; surfaces the pending `gateId`s the 2.G `gate` command points at), ✅ Done (PR #48, 2026-06-24) +(no new ADR — an additive workflow-agnostic `@relavium/db` read seam + a `@relavium/core` `parseAgent`); +and **2.L** (packaging, distribution & install verification — go/no-go #7, the last gate-closing spine PR: the +`tsup` engine-inlined ESM bundle, the bundle-closure drift guard, and the tag-triggered cross-OS install-smoke +`Release CLI` workflow), ✅ Done (PR #49, 2026-06-24) behind +[ADR-0051](../../decisions/0051-cli-distribution-thin-bundle-private-engine.md) — **closing go/no-go #7, so the +Phase-2 spine is complete and all seven Phase-3 exit criteria now hold (Phase 3 may start)**. +**Also landed — the first additive lane:** **2.S** (media host-wiring — the surface half of the multimodal +sub-spine: the `model_catalog` reader → `resolveMediaSurface` routing + the D15 catalog load-check shared by +`run`/`gate`, the content-addressed `MediaStore` de-inline to a `media://` handle, the SSRF-validated +`EgressCapability.fetch` egress, the containment-checked `save_to` write port, durable fail-cost on the terminal +events, the produced-media render surface, and the best-effort run-end host media GC), ✅ Done (PR #52, 2026-06-25) +behind [ADR-0042](../../decisions/0042-engine-media-storage-substrate-mediastore-deinline-retention.md)–[ADR-0046](../../decisions/0046-inline-media-out-via-generate-streaming-triad-deferred.md) +(no new ADR). +**Also landed — the first user-facing `AgentSession` surface:** **2.M** (`relavium chat` — the agent-first +interactive REPL over `AgentSession`: streaming tokens, tool-call annotations, the FS-scope tier + `allowedCommands` +allowlist honored, `git_commit` denied; `/exit` / `/cancel` / an input-stream EOF / raw-mode Ctrl-C all end the +session with **exit code 4** — over ONE framework-free command core driving both an `ink` TTY app and a plain +non-TTY line loop; a built-in default agent over `[chat].default_model` for a zero-config first run; durable +per-turn persistence to the shared `history.db` that round-trips via `reconstructSessionState`; the ADR-0028 +cost cap wired; model output + pasted input sanitized of terminal control sequences at the display boundary), +✅ Done (PR #54, 2026-06-26) — **no new ADR** (covered by [ADR-0024](../../decisions/0024-agent-first-entry-point-agentsession.md), +[ADR-0047](../../decisions/0047-cli-framework-commander-ink-clack.md), [ADR-0028](../../decisions/0028-workflow-resource-governance.md), +[ADR-0050](../../decisions/0050-cli-history-db-at-rest-posture.md), [ADR-0029](../../decisions/0029-tool-policy-hardening.md)). +`read_media` **input** access (D12) — which 2.S had pointed at 2.M — was **split into a dedicated, +security-reviewed follow-up** (maintainer-approved); the 2.M REPL shipped without it (tracked in +[deferred-tasks.md](../deferred-tasks.md)). +**Also landed — the rest of the agent-first chat family:** **2.N** (`relavium chat-resume` — reload + continue a +persisted session over a shared REPL), **2.O** (`chat-list` — over a new additive `SessionStore.listSessions` +read seam), **2.P** (`chat-export` + the in-REPL `/export` — session → `.relavium.yaml` scaffold, [ADR-0026](../../decisions/0026-session-export-to-workflow.md)), +and **2.Q** (`chat --json` — a headless `SessionEvent` NDJSON driver — + the one-shot `relavium agent run` with a +minimal in-house `--fixture` cassette for deterministic offline replay), all ✅ **Done (PR #55, 2026-06-26)** — +**no new ADR** — completing the agent-first CLI lane. (`agent run --input` is reserved/rejected until session +`{{ctx.*}}` prompt interpolation lands — a tracked engine follow-up in [deferred-tasks.md](../deferred-tasks.md).) +**Also landed — 2.R (the inbound MCP client):** agents now consume external MCP servers' tools across `chat`, +`run`, and one-shot `agent run`. The **`@relavium/mcp`** foundation (the SDK-fenced package, the dependency-free +JSON-Schema→Zod compiler, the fail-loud connect-all manager, the `mcp_{server}_{tool}` namespacing) is ✅ **Done +(PR #56, 2026-06-26)**, and the **host wiring** (chat/run/agent-run), the **network transports** (`http`/`sse`/ +`websocket`) behind the SSRF pre-connect floor + the per-server `allow_local_endpoint` opt-in, **named secrets** +via the isolated `mcp-secret:*` keychain namespace, the by-name `ref` registration form, and the **real-spawn +e2e** are ✅ **Done (PR #57, 2026-06-27)** — behind [ADR-0034](../../decisions/0034-mcp-client-sdk-dependency.md), +[ADR-0052](../../decisions/0052-inbound-mcp-client-package-lifecycle-registration.md), and +[ADR-0053](../../decisions/0053-mcp-network-transport-egress-security.md). It was off the M3 critical path and the +Phase-3 go/no-go (capability without gating). Residual MCP hardening — the connect-by-validated-IP dialer, +network header-auth, tool-list caching, mid-call abort propagation — is tracked in +[deferred-tasks.md](../deferred-tasks.md). The **stdio spawn is gated** as of 2.6.5 `CR-16` +([ADR-0084](../../decisions/0084-consent-before-a-local-mcp-spawn.md)): consent per machine and per resolved +declaration, on every path that can open an MCP-bearing artifact. The "import-trust" framing is retired — a +`git pull` changes a committed artifact with no import step — and only the `npx` version/integrity pinning half +remains deferred. +**Also landed — 2.J (the YAML-authoring lifecycle), the last in-phase lane:** `relavium create` (a +`@clack/prompts` wizard scaffolding an agent **or** a minimal single-agent workflow, validated against the +kind-appropriate `@relavium/shared` schema before write, dual-TTY-gated), `relavium import ` (schema- +validated copy-in with **project-global** id uniqueness), and `relavium export ` (a portable copy +**re-serialized from the validated AST** — canonical, comment-free, no provider key by construction), sharing one +`assertSlugAvailable` cross-catalog guard, ✅ **Done (PR #58, 2026-06-28)** — **no new ADR** (covered by +[ADR-0026](../../decisions/0026-session-export-to-workflow.md)/[ADR-0047](../../decisions/0047-cli-framework-commander-ink-clack.md)). + +**Phase 2 — CLI is feature-complete.** Every in-phase workstream (2.A–2.S) is merged and the published CLI is cut +as **v0.1.1**; M3 was reached at 2.K and the Phase-3 go/no-go held from 2.L. See the +[Phase 2 workstreams]() and the +[sequencing plan](#sequencing--parallelization). The full status-aware history is the +[Remaining build order](#remaining-build-order) section (its queue is now empty). + +**Phase 2.5 — CLI Consolidation & Conversational Home has started.** **2.5.A** (the spine's secure base — a +shared `assembleToolEnv({ profile, fsScopeTier, workspaceDir })` factory wired into **both** the chat and +workflow-run paths, the host-side `fs` (`realpath`+`commonpath` jail, read-only chat fail-close, single-fd +`O_NOFOLLOW`/`O_NONBLOCK` reads) and `process` arms, the advertise-filter, **EA1** `tool_unavailable`, and **EA2** +real failed-turn usage) is ✅ **Done (PR #60, 2026-06-28)**, behind +[ADR-0055](../../decisions/0055-cli-host-capability-seam-tool-environment-factory.md) — **reaching milestone +M2.5-1 (secure base)**. The `egress`/`os` arms, the `project`-tier `extraRoots` allowlist, and a write-capable +chat are deferred to **2.5.E**/[ADR-0057](../../decisions/0057-cli-chat-modes-and-per-tool-approval.md) (tracked in +[deferred-tasks.md](../deferred-tasks.md)). **2.5.B** (the bare-invocation Home) is ✅ **Done +(PR #61, 2026-06-29)**, behind [ADR-0054](../../decisions/0054-cli-bare-invocation-interactive-home.md) (Accepted): the +TTY-gated bare `relavium` opens a read-only management strip (recent sessions/runs/agents + an "Attention +required" section of pending human gates / failed runs) over a bounded, indexed `history.db` read seam, sitting +above a live prompt that graduates into an in-process chat; rendered as a single ink tree (one `useInput` owner) +with one SIGINT/SIGTERM lifecycle (clean Home exit `0`; an external signal → the conventional `128+signo`, 130/143; +the in-Home chat's exit-`4` consumed, never leaked) and bracketed paste (DECSET 2004), all while every +non-interactive path keeps the byte-for-byte help + exit-`0` meta-op ([ADR-0049](../../decisions/0049-cli-machine-output-contract.md)). +Canonically homed in [home.md](../../reference/cli/home.md). **2.5.C** (the in-app command system) is ✅ **Done +(PR #62, 2026-06-30)**, behind [ADR-0056](../../decisions/0056-cli-in-app-slash-command-system-and-manifest.md) +(Accepted): a curated **two-registry** model (the shell `COMMAND_MANIFEST` driving `commander` + `--help --json` ++ the `executeCommand` dispatch, vs the in-REPL `REPL_COMMANDS` driving a filterable `/` palette + slash commands +in **both** the chat and the bare Home — no command in both); `/help`, the `notice` output channel, `/workflows`, +`/cost`, and `/doctor` (fast tier: keychain/config/wired-tools; `--deep`: a **redacted** provider-key probe + a +**read-only** MCP-status report — a security-review decision: it reports the live session's already-connected +servers, never a fresh connect/spawn); plus the `name + args` slash dispatch and a context-aware footer hint-bar +surfacing `/ for commands`. Canonically homed in [commands.md](../../reference/cli/commands.md) + +[chat-session.md](../../reference/cli/chat-session.md). **2.5.E** (chat modes + per-tool approval + mid-turn abort) +is ✅ **Done (PR #63, 2026-07-03)**, behind +[ADR-0057](../../decisions/0057-cli-chat-modes-and-per-tool-approval.md) (**Accepted** after the mandatory +security review): the reseat-less mode system (ask / plan / accept-edits / auto on `Shift+Tab` + `/mode`), the +fail-closed per-tool `confirmAction` floor (`[y]/[a]/[n]` + a session once/always cache), the `Esc` mid-turn +abort (EA7), and the host arms closing the 2.5.A deferral — a write-capable `fs` tier + **protected paths** +(refused in every mode incl. `auto`), the SSRF-hardened `egress` arm (shared with media), and the `os` arm (now +a governed action class). Wired live into `relavium chat`, one-shot `agent run`, and the Home (each activates +the regime before its first turn). Every step passed the mandated opus + Sonnet 5 loop plus the dedicated +holistic security review (~50 findings fixed, 4 HIGH). A same-PR chat-UX follow-up also landed: a host tool +EXECUTION failure on the interactive surface (a file-not-found READ) is fed back to the model to recover +(`recoverToolFailures`, scoped to IDEMPOTENT tools via a stamped `ToolExecutionError.recoverable`; a governed / +side-effecting failure stays fail-fast) plus a static secret-free `tool_failed` hint. **With 2.5.E the CLI +Consolidation spine (2.5.A → C, E) is complete.** **2.5.D** (chat input ergonomics) is ✅ **Done (PR #64, merged +2026-07-03)** — the first experience-arm workstream: the pure-ergonomics half (`Ctrl+J` multiline, +`↑/↓` history + `Ctrl+R` reverse-search, readline motions, a shared cursor-bearing `EditorState`) plus the two +data-moving affordances behind [ADR-0061](../../decisions/0061-cli-input-layer-file-injection-and-shell-escape.md) +(**Accepted** after a two-round maintainer security review): **`@`-mention** (dir-navigable file completion that +reads through the SAME `FsCapability` `read_file` uses — jail + the expanded sensitive-read confidentiality floor ++ listing-gate + binary/size guards — and injects as UNTRUSTED, nonce-fenced, byte+line-bounded context) and +**`!`-shell** (the additive `AgentSession.runUserCommand` — EA8 — routing `!command` through the one `run_command` +boundary: `enforcePolicy([chat].allowed_commands)` BEFORE the mode-aware `confirmAction` → `spawn`/`shell:false`; +**empty-default allowlist ⇒ `!` inert**, secure-by-default, with an actionable deny hint). Each of the 5 steps +passed the mandated opus + sonnet adversarial-review loop, and the ADR-0061 mandatory security pass ran inside the +step-4/5 loops (14 findings fixed across the two `@`-mention rounds; on the `!`-shell the opus + security pass +confirmed 0 defects after adversarial verification, and the sonnet second pass caught a HIGH — a `!`-command in +flight left no host-visible busy signal, so a message typed mid-command could crash the session — now fixed with a +`shellBusy` input gate, plus a LOW type-hygiene fix). A post-implementation comprehensive review then refined the +`@`/`!` presentation to a **pending-attachment (chip) model** (an inline `@path` marker + a read-only `!`-output +preview, expanded into the SAME UNTRUSTED nonce-fenced frame only at submit — byte-identical model context, a +clean prompt), and two follow-up review passes hardened the `[chat]` allowlist resolution (exact + glob arrays are +now a **coupled unit**) and fixed a Backspace regression (ink reports the Unix physical Backspace as `key.delete`); +all recorded in the ADR-0061 "Refined at implementation" append. **2.5.F** (the ADR-0062 context commands) is ✅ +**Done (PR #65, merged 2026-07-05)** behind [ADR-0062](../../decisions/0062-context-compaction-and-cli-history-commands.md): +`/clear` (a host-level fresh-session swap across `chat`/`chat-resume`/Home, TTY-interactive only), the +`session:compacting` "Summarizing…" moment event (amending ADR-0036), and the footer context-fullness indicator — +completing compaction alongside the earlier `/compact` + `/trim` + automatic compaction. **2.5.G** (onboarding +wizard + Home `/models` + the live model catalog) is ✅ **Done (PR #66, 2026-07-07)** — its scope **expanded to +Option A**: a **live** model catalog (per-key provider discovery + a DB cache + refresh + a static/live merge) +and a complete model-pricing story (user-supplied pricing that governs cost), behind three ADRs +([ADR-0063](../../decisions/0063-cli-config-write-contract.md) config-write · +[ADR-0064](../../decisions/0064-live-model-catalog.md) live catalog · +[ADR-0065](../../decisions/0065-provider-economics-and-extensibility.md) provider economics), across 12 reviewed +steps (all landed); with it **milestone M2.5-2** is reached. The additive lane **2.5.H** (reasoning render + live-turn feedback + an actionable error taxonomy) is ✅ +**Done (PR #67, 2026-07-07)** — behind **EA6** (a new dual-envelope `agent:reasoning` stream event that *amends* +[ADR-0036](../../decisions/0036-run-loop-substrate-event-bus-and-execution-host.md); no new top-level ADR): a +host-emit of the reasoning the `@relavium/llm` seam already carries (ADR-0030), a collapsible "thinking" panel +(`/thinking` / `Ctrl+T`), the `Thinking…/Working… {elapsed}s · Esc to stop` timer, a visible `…` elision marker +(fixing the silent live-buffer head-drop), per-attempt `via {model}` failover attribution, and a static, +secret-free per-`ErrorCode` recovery hint (session-survives; a context-overflow message heuristic → `/compact`· +`/trim`). Four steps, each opus + Sonnet reviewed (3 HIGH fixed: a run-path silent-drop, a frozen Home timer, a +scrollback elision loss; + a one-shot `agent run` hint-leak). **With 2.5.E this reaches milestone M2.5-3.** The +consolidation lanes **2.5.I** (regression harness + DB concurrency hardening — `loadFull` read-txn snapshot, +`BEGIN IMMEDIATE` writes with a deterministic `SQLITE_BUSY` retry, the concurrent chat+run + cassette-chain + +perf-budget e2es, an advisory Windows CI lane) and **2.5.J** (docs-debt: the accurate unencrypted-history +posture per ADR-0050, and `NO_COLOR`/`FORCE_COLOR`/`--color` resolution) are ✅ **Done (2.5-close-out, PR #69, +2026-07-08)** — **reaching milestone M2.5-4, so Phase 2.5 is complete, merged to `main`** — landed alongside the doable-now +Batch A–E backlog (test-hardening; 2.5.H TUI polish; `AgentParseError` line/col; the ADR-0057 approval/security +batch — `[c]` reject-with-reason, non-TTY policy, SCOPE-denial recovery, Ctrl+T-in-approval, the Trojan-Source +bidi floor, behind an append-only ADR-0057 amendment; the profile-aware advertise-filter + the in-house +`.gitignore` matcher), each implement → Opus → Sonnet with a security-review pass on the approval batch. Two +`gate.ts`-resume items (`relavium budget resume` + secret re-provide) are deferred to a focused follow-up, and +the session `{{ctx.*}}` interpolation stays with the Proposed ADR-0060 (Phase-2.6); all tracked in +[deferred-tasks.md](../deferred-tasks.md). See the +[Phase 2.5 workstreams](phase-2.5-cli-consolidation.md). A **post-2.5.G model-UX follow-up** (from six +maintainer questions on model/provider/wizard behavior) then landed as a sequenced plan: `/models` key-awareness, +the onboarding-wizard live key-validation + retry UX, wizard-dynamic provider docs, **mid-session model switching** +(the `/models` reseat across `relavium chat` + the in-Home chat — [ADR-0059](../../decisions/0059-cli-mid-session-model-reseat.md), +now Accepted), and **normalized reasoning-effort control** ([ADR-0066](../../decisions/0066-normalized-reasoning-effort-control.md), +now Accepted, **merged 2026-07-07**): a provider-agnostic effort tier (`off`/`low`/`medium`/`high`/`max`) authored in +agent YAML or the `[chat].reasoning_effort` config default, each adapter mapping it to its provider's **native** tier — +**all four providers**, DeepSeek-v4's `thinking` param included (doc-verified against api-docs.deepseek.com) — gated +per-model by a host-injected capability resolver (plus a conservative id heuristic for a live-discovered model). It is +changed **interactively**: `/effort` opens a keyboard-owning tier-selector **overlay** in `relavium chat` + the in-Home +chat (§6), and the `/models` picker's **effort sub-step** either applies a **per-turn session override** in a live chat +(no reseat, §5) or — in the **bare Home** — **writes** the model + effort as the next session's config defaults (a new +global `[preferences].reasoning_effort` key), the active tier shown in the footer. Two multi-agent review passes +hardened it: the first caught the picker (wrongly) routing effort through a full model reseat — the P0 fix rebuilt it +as the ADR-mandated session-level setter; the second caught the bare-Home effort default not being re-read for a +same-process next chat — fixed to mirror the model's fresh per-chat re-read. + +Carry-over hardening is tracked in [deferred-tasks.md](../deferred-tasks.md) — Phase 2 picks +items up as it first touches each file. Notable inheritances: 1.AH's host-wiring half +(distributed across Phases 2–6), the media-egress host-side SSRF mechanism, and the +keychain no-raw-key IPC test. diff --git a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md index fbfe63d7..abc4dacc 100644 --- a/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md +++ b/docs/roadmap/phases/phase-2.5.5-hardening-and-remediation.md @@ -3709,7 +3709,7 @@ No credential values or request captures belong in tracked evidence. | 9b.1 | **Recorded split, 2026-10-25 (maintainer-approved) — container, isolated GnuPG runner and encrypt-once:** the strict USTAR dialect extracted from the archive reader with its writer promoted; the private container built from the sealed staging in a staging-relative layout, each map's digest checked against the private manifest and its size against the finalizer's record, which must name exactly the manifest's outputs and maps (the metafile must be a JSON object); an isolated public-only GnuPG runner (GnuPG ≥ 2.2.8; `--recipient-file` with no keyring or agent, and no trustdb but the empty one GnuPG may create in the listing home); a colon-listing pre-check that refuses missing, expired, revoked, unusable (v4 with cv25519 or RSA ≥ 3072 only) or unexpected recipients and secret material; `encryptOnce` (SEIPDv1 with MDC and AES-256, exactly one `ENC_TO` for the pinned subkey, the exact produced asset, a no-clobber publish, plaintext cleanup on every path); committed synthetic public fixtures, an ephemeral round-trip key, and the GnuPG prerequisite documented. Library and synthetic controls only: product paths and workflows stay unchanged. Crypto/custody security sitting. *Independently accepted on `development` at `147c2788` (2026-10-25) ([decisions](#wave-3-step-9b-preflight-decisions--2026-10-25), [checkpoint](#wave-3-step-9b1-implementation-checkpoint--2026-10-25), [acceptance](#wave-3-step-9b1-acceptance--2026-10-25)).* | | 9b.2 | **Recorded split, 2026-10-25 (maintainer-approved) — reuse, binding receipt and scope separation:** retained-ciphertext reuse without re-encryption (digest and size verified, zero GnuPG calls); the release binding receipt's schema v1, its validator and a sealer that writes only after the ciphertext exists (9d adds the origin, map-check and `ci`-run binding); the production recipient loader, absent in Wave 3; synthetic and production scope separation. Crypto/custody security sitting. *Independently accepted on `development` at `563402e6` (2026-10-26) ([decisions](#wave-3-step-9b2-preflight-decisions--2026-10-25), [checkpoint](#wave-3-step-9b2-implementation-checkpoint--2026-10-25), [acceptance](#wave-3-step-9b2-acceptance--2026-10-26)).* | | 9c | **Recorded split — offline release state controller:** permission-enforcing offline GitHub/npm controller, permissioned prebuild discovery and reservation, and every rerun, partial, conflict and expiry state, with no overwrite, rebuild or re-encryption on uncertain history. | -| 9d | **Recorded split — workflow, runbook and whole-Step-9 landing:** least-privilege actual workflow dependency graph, required artifact/platform aggregation and exact-artifact consumers, lost-staging refusal and the release runbook update. The whole-Step-9 cumulative review retains all 9a–9d changes with preservation traces into Step 8, and the dedicated archive/crypto security sitting closes the step. | +| 9d | **Recorded split — workflow, runbook and whole-Step-9 landing:** least-privilege actual workflow dependency graph, required artifact/platform aggregation and exact-artifact consumers, lost-staging refusal and the release runbook update. The whole-Step-9 cumulative review retains all 9a–9d changes with preservation traces into Step 8, and the dedicated archive/crypto security sitting closes the step. *Added 2026-11-04 (PR #96 review, maintainer ruling):* ADR-0111's "encrypt once" holds per container, not per destination — `encryptOnce` refuses a second ciphertext only in the same destination, so one container can still yield two valid ciphertexts in two destinations; 9d adds a global encrypt-once record (or an equivalent refusal) with its regression. No encrypted map is produced before 1.0 under release path B (D59), so the gap is latent until then. | | 10 | **CLI behaviour/discovery integration:** #45/#264/#265/#268/#13/#17/#18/#310/#309/#19 under ADR-0110, consuming Steps 6–9. Production owner category matrix, real root/subtree JSON help, pricing/chat aliases, non-echoing Commander refusal and signal-safe deletion consent. G15 is refuted and receives real engine/store/log preservation evidence, with no invented token coalescing. Repeat affected packed gates. | | 10a | **Recorded split, 2026-10-19 (maintainer-approved; ID coverage unchanged) — diagnostic policy and output owners:** #45/#264/#265/#268 policy wiring, the production output-owner category registry with a drift test, verbose detail whose frames come from a same-build allowlist (no second build or post-finalization write), removal of the raw stack branch with G36/G46 re-scoped to its replacement, and the PR #95 lazy-dispatch cause-wrap and purity-root fold-ins. | | 10b | **Recorded split — descriptor grammar and JSON help:** canonical chat paths and pricing spellings with working aliases and conflict refusal (#309/#18/#310), non-echoing Commander refusal with the #311 static hint family (moved here from row 17; certified in 17a), excess-argument refusal, subcommand global help (#17), real root/subtree JSON help (#13, including `help [path] --json`) and the purity guard. | @@ -3744,7 +3744,7 @@ No credential values or request captures belong in tracked evidence. | 19b | **Recorded split — whole-wave acceptance:** every disposition, canonical landings and the Proposed-section inventory, final exact-head gates, two NEW whole-wave cumulative reviews and the PR. | Steps are dependency-safe increments, not permission to parallel-edit shared hot -files. Within each row, arrows remain strict. The process/packaging sequence replaces the old fatal-before-lazy assumption: pure contracts → guarded lazy graph → native fatal integration → map/archive custody → remaining CLI behaviour. It was approved as nineteen steps; the recorded splits and the 2026-10-19 amendment make it 29 increments (see the rulings' Plan shape), with no parallel edits of shared hot files. Step 1 completes before all other +files. Within each row, arrows remain strict. The process/packaging sequence replaces the old fatal-before-lazy assumption: pure contracts → guarded lazy graph → native fatal integration → map/archive custody → remaining CLI behaviour. It was approved as nineteen steps; the recorded splits and the 2026-10-19 amendment make it 29 increments (see the rulings' Plan shape), and the 9a.2, NI and 9b splits of 2026-10-24/25 make it 32, with no parallel edits of shared hot files. Step 1 completes before all other code. Home cost is disposed or implemented before its decomposition; viewport geometry work never races editor/cache work. If review exposes a necessary smaller increment, split it with unchanged ID coverage and record the split before proceeding. @@ -10658,7 +10658,7 @@ MVP-1…MVP-10, including conversational authoring, web, the Home hub and MCP ma path B (D59); the license unchanged (D60); a scoped `development`→`main` checkpoint now (D61); the default agent may write, with `[chat].default_mode` (D62); a `run_command` approval card with remembered grants outside the repository (D63); a tiered acceptance bar (D64); six recorded splits plus #61 (D65); `current.md` reduced to live -status (D66); CR-71 un-deferred and CR-70 after the release with ADR-0096 §5 (D67); first-run defaults and one +status (D66); the CR-71 residual un-deferred and CR-70 after the release with ADR-0096 §5 (D67); first-run defaults and one keymap ADR (D68); authoring → web → Home → MCP (D69); no pre-release (D70); the parity items in the MVP (D71); no action on `0.1.1`'s #56 (D72). @@ -10702,6 +10702,8 @@ unsanitized `relavium run` output; the maintainer accepted it because the produc > status and order; only relative links were rewritten for this directory and heading levels lowered by one. > It keeps the 2026-07-26 execution plan's baseline, axes, Waves 0–1 and the PR #81 closing list — the only > definition of `#W15-1`…`#W15-24`, which code comments cite. The live order is in [current.md](../current.md). +> This first pass moved only that plan; the records it left out follow in +> [the PR #96 review correction](#moved-by-the-pr-96-review-correction--2026-11-04). ### Execution order — Phase 2.5.5 + Phase 2.6 (temporary) @@ -11120,3 +11122,612 @@ the review of `#W15-16`, fixed with it, and now the signal that test drives. redaction is per-segment (it is whole-string, and the same file says so correctly three paragraphs later); `llm-provider-seam.md` still claims model-discovery/media-poll keep a small SDK retry; ADR-0028's amendment note still says ADR-0074 §2–§5 have not landed. Each could send a future reader back toward a #91-class hole. +### Moved by the PR #96 review correction — 2026-11-04 + +> The first pass above moved only current.md's 2026-07-26 plan (its lines 502–919 at `5a328bca`). A PR #96 review +> found that the rest of its unique records had left the docs, against D66's ruling to relocate them, so they are +> moved here verbatim too: the Wave 2 and Wave 3 sections of that plan, the rows the reset edited in place (as they +> read before), and current.md's, CLAUDE.md's and AGENTS.md's status blocks. Only relative links were rewritten and +> heading levels lowered by one; a heading in brackets is added. The Phase 2.6.5 interlude and the W7/W8 records are +> in [Phase 2.6.5's record](phase-2.6.5-core-reliability-remediation.md#execution-record-moved-from-currentmd--2026-11-04), +> Phase 2's status in [Phase 2's](phase-2-cli.md#status-record-moved-from-currentmd--2026-11-04) and the +> "Not started yet" summary in [Phase 2.6's](phase-2.6-conversational-authoring.md#status-record-moved-from-currentmd--2026-11-04). + +#### Wave 2 — Shut the doors + +**Complete on `main`, 2026-10-15, merged as PR #94 (`ab08209e`).** The [merge checkpoint](#wave-2-merge-checkpoint--2026-10-15) joins all forty queue IDs, the exit/D sign-off and final correction evidence. Hostile-MCP work is reused from W4/PR #87; the remaining filesystem, secrets/config, persistence and media work is complete within its recorded limits. The queue below preserves scope and traceability. + +1. ✅ **Completed in W4/PR #87:** 2.5.5.D · project-layer MCP name collision redirecting a global secret + (G19) and MCP tool-definition poisoning (#202); see the + [W4 closing register](phase-2.6.5-core-reliability-remediation.md#w4-closing-register). +2. **MCP queue, strict — EXECUTED AS PHASE 2.6.5 `W4`, and COMPLETE there (2026-09-01, [ADR-0088](../../decisions/0088-the-mcp-boundary-is-hostile.md)).** The queue below landed in five steps, plus `#202` and `G19` from step 1 of this wave and the MCP half of `#21` from 2.5.5.E — all the same threat class and the same security sitting, so splitting them would have booked the reviewer twice. Certify the 2.5.5 ids from the `W4` closing register in [phases/phase-2.6.5-core-reliability-remediation.md](phase-2.6.5-core-reliability-remediation.md). + *The original queue, for the record:* Connect-phase timeouts on every transport + (#35, G32, #205) → transport/discovery/result size bounds against a hostile server (G33, #201, #209, #288) → + structured `serverId`/`reason` discriminants (#203, #204) → the two fail-loud violations (#206, #207) → + per-file transport-adapter tests (#297) → `CLIENT_INFO.version` (#208). These finding ids keep their home in + [phases/phase-2.5.5-hardening-and-remediation.md]() and + certify from there in step 6 below; the *work* happens in 2.6.5 alongside `CR-16` (consent before a stdio + spawn) and `CR-41` (the DNS/redirect SSRF hole), because they are the same code and the same security + sitting. Splitting them across two phases would book the hostile-MCP reviewer twice. +3. ✅ **The fs jail, ONE reviewed change set — complete on `main`, PR #94** — 2.5.5.D · sensitive-credential floor for `.kube`/`.azure`/gcloud + (#36, #39) → extract `deepestExistingReal` to `packages/shared` (#235) → **2.6.M's `project`-tier + `extraRoots`** and **2.6.N's fs-floor home-anchoring**, both *pulled forward*. `~/.relavium/tmp/` (G21) + is resolved under the same general predicate (decision D12) — otherwise `fs.ts` takes two security reviews. +4. ✅ `discoverCatalog` realpath containment (#1, accepted in Step 2); the secrets-layer test pair — OS-keychain bridge (#27) and + `readSecretFromStdin` + stdin cap (#31, #293); secret-typed `run --input` rejection (#25); the unredacted + workflow-snapshot doc + lint (#30); 2.5.5.B · `scrubSecrets` param-name broadening (#196). +5. ✅ Persistence-security: 2.5.5.C · `assertRelativePath`'s Windows drive-relative form (#113), media-store + hardening (#111, #108, #112), media `mimeType` verification (#107, #109, #110), and certification of + W7's structural session/tool-content controls (#115, Wave D15). 2.5.5.A · chat-export continuity/slug + completion and existing tool-content exclusion (#80, #81, #83, D14), plus `analyzeSecretTaint`'s + context sites (G22). 2.5.5.D · the dormant `default_headers` admission/custody gate (G20, D13). +6. ✅ **Phase 2.5.5 exit criteria 1, 2 and 3 and sub-stream D acceptance are satisfied on `main` by PR #94.** The D half of M2.5.5-2 is complete; the I half remains open. + +> Book **four security-review sittings by threat class** (fs/path-jail · secrets-at-rest · secrets-input · +> config-trust), not ~30 per-PR passes. The **hostile-MCP** sitting moved to Phase 2.6.5 with the queue-2 work; +> it covers `CR-16` and `CR-40`–`CR-42` there. Reviewer availability, not code, is this wave's critical path. + + +#### Wave 3 — Clear the ground + +**Implementation in progress — Steps 1–8 merged to `main` (PR #95, 2026-10-18); Steps 9a.1, 9a.2a, 9a.2b, NI, 9b.1 and 9b.2 accepted on `development` (2026-10-24/26), Steps 9c–19 NOT STARTED — 2026-10-26.** +ADR-0110–0113, their linked contracts and the nineteen-increment plan were approved on 2026-10-16. +[Steps 1–8 were independently accepted](#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) +on `development`, with whole Step 8 reviewed at `c83c8575`, and published as the scoped checkpoint +[PR #95](https://github.com/HodeTech/Relavium/pull/95); its [merge checkpoint](#wave-3-steps-18-merge-checkpoint--2026-10-18) is the canonical record of +what merged. The phase register owns each increment's source, tests, cumulative review rounds, security sittings and +platform/publication evidence, including historical NONCLEAN R1–R24 records, the +[previous corrective review completion](#pr-95-corrective-review-completion--2026-10-18), +the [second correction register](#pr-95-second-systematic-review-corrections--2026-10-18), +the [second corrective source-review completion](#pr-95-second-corrective-source-review-completion--2026-10-18) +(accepted at `ec0f3e10`), the [native Windows admission preparation](#pr-95-native-windows-admission-preparation--2026-10-18) +and the [supplemental fixture acceptance](#pr-95-supplemental-native-fixture-source-acceptance--2026-10-18). +Those scopes are not widened: the delta from `ec0f3e10` to the merged head is two independently accepted test-fixture +increments, documentation and the independent review's corrections at `a3e164b5`, which merged with author +verification only and were then independently reviewed after the merge +([review record](#independent-review-of-a3e164b5-and-its-follow-up-corrections--2026-10-19)); +the follow-up corrections are on `development`, not yet on `main`. Whole-wave acceptance remains Step 19. No new decision, credential, billing or provider capture gate is introduced. + +Decompose, guard and normalize every hot file a 2.6 workstream is about to land on. +**The multi-file queue is Step 1 and runs first.** The approved +[nineteen-step execution plan](#proposed-implementation-increments), +recorded as 29 increments since 2026-10-19, owns the full order and review gates: finish each increment and its fresh reviews +before advancing; no parallel edits of shared hot files. + +The groups below retain the original scope inventory and within-file dependency +arrows, not independent scheduling permission or current completion claims. The +[source-corrected preflight dispositions](#wave-3-preflight-and-proposed-execution-plan--2026-10-15) +own whether/how each original finding needs implementation or preservation. + +Two orderings here are not this plan's invention — they are the intra-phase serializations +[2.5.5 records under Positioning](#positioning): **E before I** +on `home-controller.ts`/`chat-ink.tsx` (the decomposition moves already-guarded code), and **C before E** on +`packages/db/src/client.ts` (C types the error path in Wave 1; E's `openLocalDb` rewrite lands last, here). + +- ✅ **Multi-file Step 1 accepted — 2026-10-16** ([evidence](#wave-3-step-1-acceptance--2026-10-16)) — 2.5.5.E · `closeQuietly()` to its four siblings (#22) → `displayPath` + discipline to `resolve.ts`/`catalog.ts`/`chat-export.ts` (#4, G11) → `const exhaustive: never` across the + six sites (#300, #301, G3, which also makes `relavium run` render `agent:reasoning`). +- **`chat.ts` (strict)** — 2.5.5.G · the four money/token reimplementations (#236, #237, #242, #307) → + `swapAgentModel` → `reseatAgentModel` (#218) → 2.5.5.E · SIGINT actually calling `session.abort()` (#21, #23) + → `budgetWarningText` → **2.5.5.I · decompose `chat.ts`** (#220, #244). Then **2.6.M's `allowedDomains` + chat gap (G30)**, pulled forward into `session-host.ts`. +- **`home-controller.ts` (strict)** — 2.5.5.E · `selectGatePrompter` test (G37) → the swallowed palette + rejection (#281) → **2.5.5.I · decompose `createHomeController` + the overlay discriminated union** + (#215, #243) → 2.5.5.G · the `Capability`/`Port` vocabulary split (#217, #222). + Settle #125/D26 before decomposition: the approved preflight certifies the planned scope of the real + in-Home-chat cost path and retaining chat-only scope. If a new bare-Home cost + surface is selected, wire its explicit session target before decomposition. +- **`chat-ink.tsx`** — 1,906 lines at preflight (the original queue recorded 1,795), decomposed by **neither** phase doc. + **Recommendation: decline** a new decomposition (decision D18); take only the exhaustiveness guard and the + min-terminal-size guard (#63), and hand restructuring to 2.6.M's render-v2 in Wave 5a. +- **Viewport (strict)** — 2.5.5.I · share the ADR-0069 grapheme segmenter (#58, #62, G40, G51) → + 2.5.5.G · tab expansion before width measurement (#65) → per-cluster width cache + resize debounce + (#64, #66), refined to frame coalescing in the proposed plan. Alt-screen word-wrap + (#61) retains the approved Wave 5a ownership; it is not an unapproved Wave 3 prerequisite. +- **`packages/shared` (strict)** — the `.code` ruling (G50, D10) → `findDuplicates`/`superRefine` implementation or preservation at the + schema dispositions (#89, #90, G16, G17, G18, #210, #211, #212) → `BaseEventSchema` disposition (#213) → + the `RelaviumError` base class (#219). *The ruling must precede the base class, or it is baked in permanently.* +- **`registry.ts` / `builtins.ts` / `engine.ts`** — declared-source context repair and honest unsupported-mapping disposition (#92/Accepted ADR-0113) → `ctx.fsScope` (#93) → + `ok`/`success` (#216, #221) → identifier-sort `localeCompare` pinning (#258, #259) → the `git_commit` + denial copy (#308); then the four schema length caps (#95), the `llmVisibleParams` rewrite (#96, #98), + stale doc comments (#78, #97, #214); then `#armRunTimeout` (#75), `#dispatch` dead branch (#79), + `AbortSignal` into `resolvePrompt` + the human-gate recheck (#76, G9). +- **Process/CLI boundary (strict head)** — the revised plan serializes pure flag/descriptor/owner substrate → lazy guarded runtime graph → native crash integration → map/archive custody → command behaviour. It covers 2.5.5.E · the five crash/signal net gaps (#40, #41, #42, #224, + #269), the `--help`/`--version` historical ~400 ms tax (#43, #44, #271), and the flag-system ruling (#45, #264, + #265, #268), the exit-code and `--help --json` rulings (G35, #13), durable-only logs preservation (G15 refuted), the `--input` + namespace ruling (#18, #310). **`openLocalDb` wrap-and-redact (#278, #280) lands LAST** — it touches 8+ + call sites the other queues are still moving. +- The numbered plan assigns the remaining 2.5.5.A/E/G/I tails and keeps the central + DB refactor last before whole-wave acceptance. Step 19 owns sub-stream **A, E + and G sign-offs**, the reconciled **EXIT:4 reading** (D22), and **M2.5.5-3** + closure. Full I acceptance remains in Wave 5a. + + +#### [Rows of current.md edited in place on 2026-11-04 — their wording before] + + +- **Sub-stream B, C and H acceptance sign-offs**, then **cut and publish v0.2.0**. **Closes M2.5.5-4.** + + +Roughly 60 open decisions sit across both phases: the ~14 that 2.5.5 flags **blocked** inline +([Dependencies](#dependencies)), the nine unwritten 2.6 ADRs + +| D7 | 0 | Publish v0.1.1 as-is, or supersede with v0.2.0? | v0.2.0 — ADR-0067's Node `>=22` bump is breaking for 0.x | + +| D50 | 6 | ADR row 11: adopt `ink-syntax-highlight`, or build in-house? | **Decline the dependency** per CLAUDE.md rule 2 — the markdown renderer is already in-house and untrimmed `highlight.js` is the phase's own named bundle risk | + + +This ordering is compatible with both phase files' stated dependencies. It departs from their prose in six +places, each deliberate: + +### [current.md's status block, as of 2026-10-25] + +This page tracks what is active **right now** and the immediate next concrete actions. +The full phase plan and the global milestone spine are in [README.md](../README.md). +**Phase 2.5 (CLI Consolidation) is complete** (milestone **M2.5-4**, PR #69, 2026-07-08) — its +breakdown, now historical, is in +[phases/phase-2.5-cli-consolidation.md](phase-2.5-cli-consolidation.md). +A 2026-07-19 full-project code review then opened **Phase 2.5.5 (Hardening and Remediation)**, an +interstitial phase holding the review's verified findings against the pre-2.6 codebase; it runs +**concurrently with Phase 2.6, not before it** — see +[phases/phase-2.5.5-hardening-and-remediation.md]() +and [Active now](../current.md#what-is-active-now). +**Phase 2.6 — Conversational Authoring and the First-Class CLI** is **in progress** (re-scoped +2026-07-08); its plan is in +[phases/phase-2.6-conversational-authoring.md](phase-2.6-conversational-authoring.md). +Workstream **2.6.F (platform floor + the full-screen TUI renderer)** is **merged to `main`** +(PR #74, 2026-07-11), and so is **2.6.C** (the reseat transcript-carry + the `/cost` per-model breakdown, +PR #75, 2026-07-13) — see [Active now](../current.md#what-is-active-now). **2.6.Q** (dynamic model-catalog +enrichment) is **no longer blocked**: [ADR-0071](../../decisions/0071-models-dev-as-the-model-metadata-source.md) +and [ADR-0072](../../decisions/0072-model-metadata-in-the-db-behind-a-generated-offline-floor.md) are both +**Accepted**, resolving the six open maintainer questions the phase file had cited, and the P1–P5 +implementation steps are **merged to `main`** (PR #76, 2026-07-26). P6 and the `/settings` → `/models` +visibility toggle remain open; the bundled offline snapshot was **regenerated** in PR #79 after the D3 +price ruling. + +**The unified build order across both phases is in +[Execution order — Phase 2.5.5 + Phase 2.6](../current.md#execution-order--phase-255--phase-26-temporary) below** — a +temporary section, deleted when both phases close. Its baseline step is discharged (PRs #76 and #77 merged) +and Wave 0 is merged (PR #82), as is Wave 1 (PR #83). The interlude's `W2` (core reliability, six items) is +merged too (PR #85, 2026-08-28). + +### Wave 2 merged — Shut the doors + +**Wave 2 merged to `main` on 2026-10-15 as [PR #94](https://github.com/HodeTech/Relavium/pull/94), +merge commit `ab08209e`. All eight steps and the maintainer corrections are complete.** +The [merge checkpoint](#wave-2-merge-checkpoint--2026-10-15) +joins the forty item dispositions, four new security sittings plus W4 revalidation, +original whole-wave acceptance and all subsequent correction scopes. Existing incoming +navigation anchors are retained; dated implementation checkpoints remain historical. + +Completed work covers invocation-authorized filesystem boundaries and scratch-policy +composition; bounded secret stdin, early admission and private bridge diagnostics; +faithful-snapshot warnings, structural session privacy and export completion; verified +bounded media/CAS/native lifecycles; and the interim provider-header gate. Maintainer +corrections include runner-level error rejection, valid Opus/Vorbis publication, +FLAC structural consistency, generated-result streaming and finite pin-error provenance. + +Final reviewed correction source `4cdfe011` has two NEW independent complete clean +cumulative rounds covering all 67 correction paths / 188 hunks / 24 complete new files. +These scoped rounds do not claim a new review of every original PR path. Full local +CI, ordinary/enforced coverage and all nine jobs in each PR/push CI workflow pass; +SonarCloud and selected native Windows/Node-floor controls pass. Local coverage records +588 files / 12,983 passing tests / 20 explicit skips; CI records 12,982 / 21 with its +established extra CI-only skip. Complete physical controls are independently inspected. +The merge tree matches the reviewed source; no new merge-commit execution is inferred. + +**Satisfied on `main`:** Phase 2.5.5 exit criteria 1–3 and sub-stream D acceptance. +**Still open:** M2.5.5-2's I half, whole Phase 2.5.5/2.6, M/N production lifecycle, +future live headers and existing approved residuals. Wave 2 has no remaining implementation, +review, publication, ADR approval, provider capture, credential, billing or user-decision gate. +The [deferred register](../deferred-tasks.md) retains its existing limits; closure adds no deferral. + + +### Wave 3 Steps 1–8 merged — PR #95 + +**Wave 3 Steps 1–8 merged to `main` on 2026-10-18 as [PR #95](https://github.com/HodeTech/Relavium/pull/95), +merge commit `6d2a7372` (22:44 UTC). Wave 3 itself is not complete: Steps 9–19 had not started at +that merge (Steps 9a.1, 9a.2a and 9a.2b are now accepted on `development`, the native-identity increment NI and Steps 9b.1 and 9b.2 are accepted too, and Steps 9c–19 have not started) and whole-wave acceptance +remains Step 19.** The [merge checkpoint](#wave-3-steps-18-merge-checkpoint--2026-10-18) +joins the completed scope, every review attribution, the independent review's dispositions and the +publication state observed at merge. It was the maintainer-authorized, scoped publication checkpoint. + +On `main` now: close handling, safe source labels and exhaustiveness guards (Step 1); shared admission and +diagnostics (2); secret provenance at public sinks, ADR-0113 (3); the typed error/envelope substrate (4); tool and +engine tails (5); the pure process/discovery substrate, ADR-0110 (6); the lazy runtime, finite compiled graph and +exact-tarball consumers, ADR-0111 (7); and creation-owned native process integration (8). + +Three points were recorded rather than papered over; all three are now resolved. The merged head `a3e164b5` carried +corrections from a further independent review (a secret-scrubbing production rule, two behaviour-neutral import moves, +tests and causal-control pins) with **author verification only** at merge; a post-merge independent review — four +rounds and a security sitting, with fixes in follow-up commits — accepted them, as the +[review record](#independent-review-of-a3e164b5-and-its-follow-up-corrections--2026-10-19) shows. CI had not finished when the merge +happened (the required `lint · typecheck · test` job was still running); the three runs later completed `success`, +15 of 15 jobs each, as the [merge checkpoint](#publication-state-observed-at-merge) +records. And SonarCloud's quality gate, which had failed on `main` since at least PR #90 because its dispositions lived +only on PR branches, was cleared on 2026-10-19 (zero open vulnerabilities, gate OK). Open findings remain, including two +security residuals that predate the follow-up — an `api-key:Bearer ` token leak and unmasked userinfo with a raw +`?`/`#` — in [deferred-tasks.md](../deferred-tasks.md#pr-95-independent-review--unactioned-findings-2026-10-18). + + +### Next work — Wave 3 — Clear the ground + +Wave 3 implementation continues on `development`: Steps 1–8 are merged. A pre-implementation document/source review +of Steps 9–19 and the maintainer's rulings are recorded in the +[2026-10-19 rulings](#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) +(no new ADR; 29 recorded increments). **The PR95 increment is closed on `development`** — the PR #95 +scrubber residuals, including the two security residuals — under the maintainer's 2026-10-21 masker acceptance +bar and round cap, as its +[implementation checkpoint](#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20) records: rounds 1–10 were NONCLEAN; the high and +medium findings through round 9 are corrected, and after round 10, the last round, only GEN-R10-1, which the +maintainer counted as high on 2026-10-22, is corrected (in the summary, by the author alone). Its other findings +are recorded in +[deferred-tasks.md](../deferred-tasks.md#pr-95-masker-edge-cases--approved-deferrals-2026-10-21-and-2026-10-22). **Step 9a.1**, the first increment of the release map/custody substrate (#46, ADR-0111), is independently accepted on `development` at `cf97b7f1` (2026-10-24): its round 13 reported no blocker, high or medium finding and its security sitting was clean, and the round's low findings are recorded in deferred-tasks.md ([acceptance](#wave-3-step-9a1-acceptance--2026-10-24)); Step 9a.2 is split into 9a.2a and 9a.2b, and Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24) after three rounds, the third with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)); Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24) after two rounds, the second with no blocker, high or medium finding and no regression ([acceptance](#wave-3-step-9a2b-acceptance--2026-10-24)), and the native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25) after one round with no blocker, high or medium finding and no regression and a clean security sitting ([acceptance](#wave-3-ni-acceptance--2026-10-25)), its Windows clause open until three green advisory Windows runs; Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](#wave-3-step-9b-preflight-decisions--2026-10-25)), and Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is independently accepted on `development` at `147c2788` ([acceptance](#wave-3-step-9b1-acceptance--2026-10-25)), and Step 9b.2 (the release binding receipt v1, verified ciphertext reuse, the absent production recipient loader and scope separation) is independently accepted on `development` at `563402e6` ([acceptance](#wave-3-step-9b2-acceptance--2026-10-26)), with Step 9c next; the substrate's +9a.1–9d increments carry a dedicated archive/crypto security sitting. Steps 10–19 follow the +[increment table](#proposed-implementation-increments) and end with +whole-wave acceptance at Step 19. `development` is pushed after each accepted increment; PR #95 no longer bounds the +review scope of new commits. See the [Wave 3 queue](../current.md#wave-3--clear-the-ground). + + +### Where we are + +**Phase 1 — Engine and LLM is COMPLETE** (milestone **M2** reached, PR #27, 2026-06-16; +all workstreams 1.A–1.AH merged through PR #38, 2026-06-21). The engine runs end-to-end: +YAML parse → DAG → run loop → node execution (agent + the six non-agent handlers) → +checkpoint/resume → node retry → provider failover, with per-attempt cost tracking and a +gap-free `sequenceNumber`. The `@relavium/llm` seam is frozen and proven behind a +provider-agnostic `LLMProvider` interface with all three adapters (Anthropic, +OpenAI/DeepSeek, Gemini) and the `FallbackChain` runner. Two additive sub-spines also +completed: the agent-first entry point (`AgentSession` multi-turn + session persistence + +export-to-workflow, Lane C / 1.m5, 1.V–1.AA) and multimodal I/O (media input + engine +plumbing + inline & async output generation + generative adapters, 1.m6, 1.AD–1.AH). See +[Phase 1 detail](phase-1-engine-and-llm.md), the [decision index](../../decisions), +and the [reference specs](../../reference). + +> **Repository checkpoint (recorded 2026-07-26; publication status corrected 2026-10-13):** +> (1) The CI `ci` job became a required check under `main` branch protection, alongside a +> `v*` tag ruleset limiting tag creation to maintain/admin. The optional `TURBO_TOKEN`/ +> `TURBO_TEAM` secrets for cross-runner remote cache were recorded as unset at that checkpoint; +> this does not assert their current configuration. (2) The former v0.1.1 publish-pending note +> was stale: the [npm registry](https://registry.npmjs.org/relavium) records `relavium@0.1.1` +> published on 2026-06-28. This resolves the package-publication status, without asserting the +> current `NPM_TOKEN`, npm 2FA or GitHub release-workflow state. The release procedure remains +> [ADR-0051](../../decisions/0051-cli-distribution-thin-bundle-private-engine.md) / +> [release-a-surface.md](../../runbooks/release-a-surface.md). + + +### What is active now + +#### 2026-07-19 full-project code review — triaged, Phase 2.5.5 opened + +A multi-agent code review of the whole tree at HEAD `f88b0e8` ran while the toolchain was green +throughout (lint, typecheck, and all 226 test files passing) — every finding is a defect a green +build cannot catch. 377 findings went through two-lens adversarial verification (one agent trying +to refute each finding against the code, one auditing its materiality); 373 survived and were +triaged into 205 work items. The review's central conclusion: the gap to first-class is a gap in +**propagation**, not competence — the correct pattern almost always already exists in the repo and +fails to reach the next site (a redaction helper not called twenty lines away, terminal +sanitization built for chat and never carried to `relavium run`, `withBusyRetry` applied to every +writer but the three oldest, cancel-wins implemented in four node handlers and omitted in the +fifth). That makes the remediation cheap and low-risk: the reference implementation is already +in-tree. + +Disposition: 176 work items opened the new **[Phase 2.5.5 — Hardening and Remediation]()** +(engine/LLM/persistence hardening, MCP/secrets, the CLI operator-safety net, TUI terminal safety, +docs accuracy, CI/test-infrastructure, and hygiene), running **concurrently with Phase 2.6, not +before it** (327 findings); 27 findings (16 work items) were folded directly into the charter of an +already-open 2.6 workstream; 16 findings (10 work items) were already tracked in +[deferred-tasks.md](../deferred-tasks.md) and were re-confirmed rather than duplicated; 3 needed no +action per the finding's own argument. Severity of the 373 triaged findings: 3 critical, 48 high, 134 +medium, 155 low, 33 polish. + +The three critical findings, visible here so they aren't buried in the phase file: + +- **A tool-approval preview that is not secret-redacted** (`packages/core/src/tools/registry.ts`) + — the persisted, `--json`-visible `ToolActionPreview` echoes a command/path verbatim instead of + going through the same redaction its sibling input-sanitizer already applies. +- **A transient `history.db` write failure silently drops a chat turn, then crashes the session** + (`apps/cli/src/chat/persister.ts`) — an async write failure surfaces after the reply was already + shown to the user, as an unhandled rejection. +- **Missing terminal-output sanitization on the `relavium run` TUI** (`apps/cli/src/render/tui/RunApp.tsx`) + — `stripTerminalControls`/`sanitizeInline` is applied at nearly every other dynamic-text display + boundary but never reaches this one, leaving an ANSI/OSC/Trojan-Source injection hole. + + +### [CLAUDE.md's status block, as of 2026-10-26] + +**Status: Phase 1 is complete** (2026-06-21) — the engine runs end-to-end behind the +`LLMProvider` seam with all three adapters, `AgentSession`, multimodal I/O, and the full +run-loop. **Phase 2 (CLI) is feature-complete** (published as v0.1.1): +`relavium chat` / `chat-resume` / `agent run`, `relavium run` with `--json`, the +inbound MCP client, and the full YAML lifecycle (`create`/`import`/`export`). +**Phase 2.5 (CLI Consolidation) is complete** (M2.5-4, PR #69, 2026-07-08): the bare-invocation +Home, the two-registry command system (`/` palette + shell commands), per-tool +approval/modes (ask/plan/accept-edits/auto), `@`-mention file injection, context +compaction, the onboarding wizard + live model catalog, reasoning rendering, and regression +hardening. **Phase 2.6 (Conversational Authoring and the First-Class CLI) is in progress** — two +workstreams are merged to `main`: **2.6.F** (the platform floor + the full-screen TUI renderer, +PR #74, 2026-07-11) and **2.6.C** (the reseat transcript-carry + the `/cost` per-model breakdown, +PR #75, 2026-07-13). The phase is a full-screen Home-managed CLI with conversational workflow authoring, management +browsers, competitor-breadth tools, settings/theming/`en`+`tr` localization, and the +run-ops resume follow-up. + +**The interlude between remediation Waves 1 and 2 is complete: Phase 2.6.5 (Core reliability), +49 of 51 items closed on `main`, with approved CR-70/93 and scoped deferrals retained. +W8 merged on 2026-10-11 as PR #92; W7 had restored the non-deferrable criterion** — `W0` (PR #82, 2026-08-11), `W1`, the eight P0 blockers plus `CR-92`, merged +2026-08-24 (PR #83) behind [ADR-0078](../../decisions/0078-ordered-durable-append-and-the-terminal-outbox.md)–[ADR-0084](../../decisions/0084-consent-before-a-local-mcp-spawn.md): +ordered durable append, cross-process run ownership, the durable effect journal, untrusted compaction summaries, +the stream-grammar seam obligation, engine-side input admission and resume identity, and consent before a local +MCP spawn. **`W2` — liveness and deadlines — merged 2026-08-28 (PR #85)** behind +[ADR-0085](../../decisions/0085-the-node-executor-owes-liveness-and-the-engine-enforces-it.md): the authored +agent `timeout_ms` now bounds the node, every media call is bounded, a resume re-arms the run cap and a +pending gate at their REMAINING time rather than renewing them (the node bound is the deliberate exception — +ADR-0085 §2 records why), and an executor that ignores its signal can no longer leave a run without a +terminal. **`W3` — resource governance and bounds — merged 2026-08-30 (PR #86) behind +[ADR-0086](../../decisions/0086-absolute-admission-ceilings-on-authored-values.md)**: authored values carry +absolute admission ceilings, node output / workflow state / durable events are size-bounded, and a long-lived +engine no longer retains every finished run. **It merged with a live blocker** — an un-pulled event stream +drops the terminal event, breaching ADR-0036's gap-free contract — plus nine verified findings, all recorded +in the `W3` residuals of [deferred-tasks.md](../deferred-tasks.md). +[ADR-0087](../../decisions/0087-consumed-streams-size-bounds-and-run-retention.md) records the fix and, after a +review that refused acceptance until nine dated corrections existed, was **Accepted (2026-09-04) with its §1 +unimplemented**. That blocker is now closed by W8/PR #92. Its §3 turned out to be implemented already, in `W5`. **`W4` — the hostile MCP boundary — merged 2026-09-01 (PR #87) +behind [ADR-0088](../../decisions/0088-the-mcp-boundary-is-hostile.md)**: every MCP call is bounded and +cancellable, `http`/`sse` connect by validated pinned IP, a redirect is refused, a remote `websocket` is +refused at admission, a server's ingress is bounded at two levels, and its tool DEFINITIONS are treated as +untrusted. No new product surface in either wave — only the invariants an existing surface already claims. +A systematic review of PR #87 found **five merge blockers** in `W4` — all reproduced and fixed before it +merged; the phase doc's `W4` section records what they were and what the pattern says. **`W5` — media correctness — MERGED 2026-09-02 (PR #88)** behind +[ADR-0089](../../decisions/0089-media-correctness-four-boundaries.md) and +[ADR-0090](../../decisions/0090-a-continuation-token-rides-the-part-it-belongs-to.md): `read_media` delivers +over the media-input rail on a marked, fenced synthesized message; tool and attachment capability is gated on +the MODEL; Gemini's function-call continuation token rides the part it belongs to; a media body streams from +the network into the store under a size ceiling, an idle deadline and the run signal; a `url` output is pinned +to a content-addressed handle at first resolution; and a missing media rate is unpriced rather than a price of +zero. A systematic maintainer review of the branch returned **six merge blockers**, all reproduced and fixed +before it merged. Two items closed one half of a two-part obligation and say so; every residual is written out +rather than left inside a checked box. **`W6` — authoring correctness — MERGED 2026-09-04 (PR #89)** behind [ADR-0091](../../decisions/0091-first-means-first-declared-not-first-to-finish.md)–[ADR-0094](../../decisions/0094-a-tool-grant-is-checked-when-the-plan-is-built.md): +`merge_strategy: first` means first DECLARED and the plan field no longer claims otherwise; a widened tool grant +is refused when the plan is BUILT rather than mid-run; an expression sees only its transitive dependency closure +and a literal out-of-closure `run.outputs` read is refused at parse (as is `edges[].condition`, which nothing +ever read); and an authored `output_schema` is compiled at parse in an allowlist-strict mode and enforced at run +time — with **no new dependency**, because the JSON-Schema→Zod compiler the deferral claimed we needed already +existed in `packages/mcp` and simply moved to `@relavium/shared`. Seven internal review rounds found that **most defects +were in the FIXES rather than the code they repaired** — the branch-order search was wrong three times, the +expression scan's no-false-refusal claim was false in eight ways, and a stand-down gate silently removed the +check it was added to protect; each is a dated correction inside its own ADR. A systematic maintainer review +then returned **two merge blockers**, both breaking a headline claim of the wave — `required` was not enforced +when a property's schema constrained nothing, and the expression scanner's regex was quadratic and synchronous +on a parser that accepts 2 MiB — plus eight further High findings, all reproduced and fixed before the merge. + + +**Between `W6` and `W7`, the two remaining non-deferrable items closed (2026-09-06)**: `CR-73` — `invoke_agent` +was advertised to every model granted it while `ctx.invokeAgent` is wired nowhere in the tree, so the +advertise-filter now checks a tool's declared dispatch DELEGATE and not only its `ToolHost` arm (`read_media` +rides along, for the same reason and by `CR-50`'s own argument) — and `CR-80` — a rejected custom provider +`base_url` was caught and the DEFAULT adapter left standing, which is the official API, so a drifted config +silently sent prompts and keys to `api.openai.com`; it now installs a refusing adapter naming the URL's shape, +never its credentials. The test that was supposed to pin `CR-80` asserted only `toBeDefined()`, which a +refusing adapter satisfies too, so it passed either way — the finding under the defect was the test. +[ADR-0087](../../decisions/0087-consumed-streams-size-bounds-and-run-retention.md) was also **accepted +(2026-09-04) with nine dated corrections** after a review refused it as written; §1 was still unimplemented +at that checkpoint, while §3 turned out to have shipped in `W5` under a different item number. +**W8 merged to `main` on 2026-10-11 as PR #92 (`ef0ffa54`); Phase 2.6.5 is complete with approved deferrals.** +All seven steps, CR-81/82 and ADR-0087 §1 are complete on `main`; the register is **49/51**. +The [merge checkpoint](phase-2.6.5-core-reliability-remediation.md#w8-merge-checkpoint--2026-10-11) +joins the original whole-wave acceptance, all maintainer-correction dispositions and final reviewed-head +`3d4491df` PR/push CI plus SonarCloud success. The ordinary/enforced-coverage inventory is +511 files / 11,026 passing tests / 12 established skips; four wire and 23 usage mechanisms pass. +Every correction increment has two NEW independent complete clean cumulative rounds after its final fix; +scoped correction reviews are not a new whole-PR certification. Fresh full local CI and bare/enforced +coverage at `6139563e` retain their separate source attribution. No merge-commit execution is inferred. +Two nonblocking Sonar complexity suggestions remain visible. Historical counts and pending states +throughout the registers describe their original checkpoints. + +CR-70 (cross-turn tool-history carrying), CR-93 (tenant-scoped mutable state), CR-95's long continuation +and existing exact-capacity/scoped residuals retain their approved limits in +[deferred-tasks.md](../deferred-tasks.md). No new deferral, provider capture, billing, +credential or W8 user decision is pending. The joint [Wave 2 — Shut the doors](../current.md#wave-2-merged--shut-the-doors) merged to `main` on 2026-10-15 as PR #94; +next is [Wave 3 — Clear the ground](../current.md#wave-3--clear-the-ground). +Phase 2.5.5 and Phase 2.6 remain in progress. + +**W7 merged to `main` on 2026-10-10 as PR #90, including final whole-wave Step 12.** +Steps 1–11, all five genuine overflow records, owned measured requests and consolidated provider/MCP/host +lifetimes have independent acceptance. ADR-0095–ADR-0103 are Accepted. The six W7 register items are closed +(47/51 at the W7 merge); PR #90 is merged (`46a6a701`); W7 has no open implementation or review gate. The +[closing register](phase-2.6.5-core-reliability-remediation.md#w7-closing-register--2026-10-10) +joins the accepted scopes, causal controls and systematic-review dispositions. Existing approved deferrals +stay in [deferred-tasks.md](../deferred-tasks.md); W8 subsequently merged in PR #92 and closed the phase with approved deferrals. No user decision, +billing setup, credential or provider capture is pending. + +**Post-closure corrections are independently accepted, 2026-10-10:** core, CLI/DB and tooling/docs +each have two NEW clean complete cumulative review rounds. The supplemental publication-tool +correction also has two NEW complete clean rounds. The [post-closure register](phase-2.6.5-core-reliability-remediation.md#post-closure-systematic-review--2026-10-10) +joins every report disposition, new permanent/causal evidence and current gates. Published-head checks +passed at reviewed head `69ac80b0` before PR #90 merged. No W7 implementation, provider, +billing, key or user decision is pending. The [merge checkpoint](phase-2.6.5-core-reliability-remediation.md#w7-merge-checkpoint--2026-10-10) +records W7 closure. W8 subsequently merged in PR #92; the register is 49/51 on `main`. +Existing approved limits are unchanged; W8 has no open implementation or landing gate. + +**Joint Wave 2 merged to `main` — 2026-10-15, PR #94 (`ab08209e`).** +All eight implementation steps and the maintainer corrections are complete on `main` +under Accepted ADR-0105–0109. The [merge checkpoint](#wave-2-merge-checkpoint--2026-10-15) +joins forty item dispositions, security sittings, the final reviewed source `4cdfe011`, +two NEW clean complete cumulative correction rounds and successful PR/push CI, +coverage, native-platform and independently inspected physical controls. Earlier +whole-wave and correction reviews retain their exact source scopes; no new whole-PR +review or merge-commit execution is inferred. Detailed review records remain local +and Git-ignored. + +Exit criteria 1–3 and the D half of M2.5.5-2 are satisfied on `main`. The I half, +whole Phase 2.5.5/2.6, full M/N production lifecycle, future live provider headers +and existing approved deferrals remain open in their canonical homes. No Wave 2 +implementation, review, publication, ADR approval, provider capture, credential, +billing or user-decision gate remains. Next is [Wave 3 — Clear the ground](../current.md#wave-3--clear-the-ground), +whose ADR-0110–0113/contracts/nineteen-increment plan were approved on 2026-10-16. +Its serialized multi-file Step 1 is independently accepted on `development` +(2026-10-16); Step 2 is also accepted after two NEW clean complete cumulative rounds +following its historical-reader corrections. Step 3 (ADR-0113 secret provenance) +is independently accepted after two NEW complete cumulative clean rounds and a +separate clean security sitting. Step 4 (error/envelope substrate) is independently +accepted after two NEW complete cumulative clean rounds. Step 5 (tools and engine +tails) is independently accepted after two NEW complete cumulative clean rounds +following its registry-documentation correction and a separate clean security +sitting. Step 6 (pure process/discovery substrate) is independently accepted after +two NEW complete cumulative clean rounds and a separate clean security sitting. +Step 7a (lazy runtime and finite compiled graph) is independently accepted after +two NEW complete cumulative clean rounds and a separate clean security sitting. +Step 7b and whole Step 7 are independently accepted at `c836ffe6` after two NEW +complete cumulative clean rounds, a clean security sitting and successful exact-tarball +Linux/macOS/Windows plus Node 22.0.0 consumers. Broader CI retains its separately +attributed status. Step 8a implements origin adapters and an inert native foundation; +it is independently accepted at `e0a5ae19` after two NEW complete cumulative +clean rounds and a separate clean security sitting. +Step 8b (creation-owned native processes) is independently accepted at +`e16a7659` after two NEW complete cumulative clean rounds and a separate clean +security sitting following its canonical-reference correction. Its unchanged +runtime/control evidence and successful full CI plus all four same-tarball +platform consumers remain attributed to `ab50be11`, with independently verified +source/dependency equivalence. Step 8c workflow/preparation/terminal integration +is independently accepted at `dbab25d8` after two NEW complete cumulative clean +rounds, a separate clean security sitting and successful full CI. All four +same-tarball platform consumers pass at that reviewed source. +**Step 8d and whole Step 8 are independently accepted on `development` at +`c83c8575` — 2026-10-17.** Two NEW complete cumulative CLEAN rounds (R25/R26) +each cover all 342 paths / 1,842 ordered hunks after the final fixture correction. +A NEW separate clean security sitting and all fifteen full-CI jobs pass at that +reviewed source, as do all four actual same-tarball platform consumers. Original +failed reviewer-preparation records and source/native/platform limits remain +explicit; source CI does not certify later documentation commits. The +[acceptance register](#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) +joins exact evidence and historical corrections. Steps 1–8 are accepted; +**Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 +itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now +accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI and Steps 9b.1 and 9b.2 are accepted too, and Steps 9c–19 have not started) and whole-wave acceptance remains Step 19. +The [merge checkpoint](#wave-3-steps-18-merge-checkpoint--2026-10-18) +records what merged and its review attributions. The final head `a3e164b5` (corrections from a +further independent review, including a secret-scrubbing rule) merged with author verification +only and was then independently reviewed (four rounds and a security sitting, with follow-up +fixes on `development`, not yet on `main`); CI for it and the merge commit finished green, and SonarCloud's `main` +gate (red since at least PR #90) was cleared on 2026-10-19. A pre-implementation review of Steps 9–19 +and the maintainer's [2026-10-19 rulings](#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) +give every open PR #95 finding an owner and record 29 increments (no new ADR). The PR95 increment +(the scrubber residuals, including both security residuals) is closed on `development` under the +maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were +NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last +round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25): round 1 reported no blocker, high or medium finding and no regression, with a clean security sitting; every product file and directory identity check now compares exact bigints, its low findings are recorded in deferred-tasks.md, and its Windows clause closes after three green advisory Windows runs ([acceptance](#wave-3-ni-acceptance--2026-10-25)). Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](#wave-3-step-9b-preflight-decisions--2026-10-25)); Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is independently accepted on `development` at `147c2788` (2026-10-25): rounds 1–3 were not accepted (one, one and two mediums, each corrected), and round 4 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](#wave-3-step-9b1-acceptance--2026-10-25)). Step 9b.2 (the release binding receipt v1, verified ciphertext reuse, the production recipient loader, absent in Wave 3, and scope separation; synthetic recipients only) is independently accepted on `development` at `563402e6` (2026-10-26): an internal rigor pass and rounds 1–3 were not accepted (mediums each time, all corrected), and round 4 reported no blocker, high or medium finding and no regression, its security sitting lows only; its low findings are recorded in deferred-tasks.md ([acceptance](#wave-3-step-9b2-acceptance--2026-10-26)). Step 9c is next. `development` is pushed after each accepted increment. No provider, +credential, billing or new ADR approval is pending for the approved implementation plan. + +### [AGENTS.md's status block, as of 2026-10-26] + +**Status: Phase 1 is complete; Phase 2 (CLI) is feature-complete (published as v0.1.1); +Phase 2.5 (CLI Consolidation) is complete (M2.5-4, 2026-07-08); Phase 2.6 +(Conversational Authoring and the First-Class CLI) is in progress, with an interlude — +Phase 2.6.5 (Core reliability remediation) — completed between remediation Waves 1 and 2: +49 of 51 items closed on `main`, with approved deferrals retained; W8 merged 2026-10-11 in PR #92 (`CR-96` and `CR-97` were opened 2026-09-14, `CR-98` on 2026-09-18; `W7` merged to `main` on 2026-10-10 (PR #90), with ADR-0095–ADR-0103 +accepted) — all eight P0 blockers (ADR-0078–ADR-0084, merged 2026-08-24), the `W2` +liveness-and-deadlines wave (ADR-0085, merged 2026-08-28 as PR #85), `W3` resource governance and bounds (ADR-0086), merged 2026-08-30 as PR #86 **with a live blocker and nine open findings at that merge** — the blocker is now closed by W8/PR #92; scoped residuals remain in docs/roadmap/deferred-tasks.md — and `W4`, the hostile MCP boundary (ADR-0088), merged 2026-09-01 as PR #87 — a systematic review of the branch found five merge blockers, all reproduced and fixed before it merged. `W5`, media correctness (ADR-0089 + ADR-0090), **merged 2026-09-02 as PR #88** — a systematic review of that branch returned six merge blockers, all reproduced and fixed before the merge. `W6`, authoring correctness (ADR-0091–ADR-0094), **MERGED 2026-09-04 as PR #89** — `merge_strategy: first` means first DECLARED, a widened tool grant is refused when the plan is BUILT, an expression sees only its transitive closure with a literal out-of-closure read refused at parse, and an `output_schema` is compiled at parse and enforced at run time with **no new dependency**. Seven internal review rounds found that most defects were in the FIXES rather than the code they repaired; a systematic maintainer review then returned two merge blockers, both breaking a headline claim of the wave, plus eight further High findings — all reproduced and fixed before the merge. Each is a dated correction inside its own ADR. +**W7 merged to `main` on 2026-10-10 as PR #90, including final whole-wave Step 12.** +Steps 1–11, all five genuine overflow records, ADR-0102 request ownership and consolidated ADR-0103 +host integration have independent acceptance. ADR-0095–ADR-0103 are Accepted. All six W7 register +items are closed (47/51 at the W7 merge); PR #90 is merged (`46a6a701`); W7 has no open implementation or review gate. The +[closing register](phase-2.6.5-core-reliability-remediation.md#w7-closing-register--2026-10-10) +joins evidence and every submitted review disposition. W8 subsequently merged in PR #92; approved deferrals remain. +No user decision, billing setup, credential or provider capture is pending. +**W8 merged to `main` on 2026-10-11 as PR #92 (`ef0ffa54`); Phase 2.6.5 is complete with approved deferrals.** +All seven steps, CR-81/82 and ADR-0087 §1 are complete on `main`; the register is **49/51**. +The [merge checkpoint](phase-2.6.5-core-reliability-remediation.md#w8-merge-checkpoint--2026-10-11) +joins original whole-wave acceptance, maintainer corrections and final reviewed-head `3d4491df` +PR/push CI and SonarCloud success. Published ordinary/enforced coverage records 511 files / +11,026 passing tests / 12 established skips; all four wire and 23 usage mechanisms pass. +CR-70/93, CR-95's long continuation and exact-capacity/scoped residuals remain approved deferrals. +The joint [Wave 2 — Shut the doors](../current.md#wave-2-merged--shut-the-doors) merged to `main` on 2026-10-15 as PR #94; +next is [Wave 3 — Clear the ground](../current.md#wave-3--clear-the-ground). +Phase 2.5.5 and Phase 2.6 remain in progress. No W8 implementation or landing gate remains. +**Post-closure corrections are independently accepted, 2026-10-10:** core, CLI/DB and tooling/docs +each have two NEW clean complete cumulative review rounds. The supplemental publication-tool +correction also has two NEW complete clean rounds. The [post-closure register](phase-2.6.5-core-reliability-remediation.md#post-closure-systematic-review--2026-10-10) +joins every report disposition, new permanent/causal evidence and current gates. Published-head checks +passed at reviewed head `69ac80b0` before PR #90 merged. No W7 implementation, provider, +billing, key or user decision is pending. The [merge checkpoint](phase-2.6.5-core-reliability-remediation.md#w7-merge-checkpoint--2026-10-10) +records W7 closure. W8 subsequently merged in PR #92; the register is 49/51 on `main`. +Existing approved limits are unchanged. + +**PR #92 maintainer corrections:** the [correction register](phase-2.6.5-core-reliability-remediation.md#w8-pr-92-systematic-review-corrections--2026-10-11) +records all four increments and the subsequent publication correction accepted after two NEW +clean complete cumulative rounds each. Fresh full CI and bare/enforced coverage pass at `6139563e`, +including four wire and 23 usage controls. +Final reviewed head `3d4491df` passed PR/push CI and SonarCloud before PR #92 merged. +No merge-commit execution or new whole-PR review is inferred. Two nonblocking Sonar complexity +suggestions remain visible; no new deferral or decision is added. + +**Joint Wave 2 merged to `main` — 2026-10-15, PR #94 (`ab08209e`).** +All eight implementation steps and the maintainer corrections are complete on `main` +under Accepted ADR-0105–0109. The [merge checkpoint](#wave-2-merge-checkpoint--2026-10-15) +joins forty item dispositions, security sittings, the final reviewed source `4cdfe011`, +two NEW clean complete cumulative correction rounds and successful PR/push CI, +coverage, native-platform and independently inspected physical controls. Earlier +whole-wave and correction reviews retain their exact source scopes; no new whole-PR +review or merge-commit execution is inferred. Detailed review records remain local +and Git-ignored. + +Exit criteria 1–3 and the D half of M2.5.5-2 are satisfied on `main`. The I half, +whole Phase 2.5.5/2.6, full M/N production lifecycle, future live provider headers +and existing approved deferrals remain open in their canonical homes. No Wave 2 +implementation, review, publication, ADR approval, provider capture, credential, +billing or user-decision gate remains. Next is [Wave 3 — Clear the ground](../current.md#wave-3--clear-the-ground), +whose ADR-0110–0113/contracts/nineteen-increment plan were approved on 2026-10-16. +Its serialized multi-file Step 1 is independently accepted on `development` +(2026-10-16); Step 2 is also accepted after two NEW clean complete cumulative rounds +following its historical-reader corrections. Step 3 (ADR-0113 secret provenance) +is independently accepted after two NEW complete cumulative clean rounds and a +separate clean security sitting. Step 4 (error/envelope substrate) is independently +accepted after two NEW complete cumulative clean rounds. Step 5 (tools and engine +tails) is independently accepted after two NEW complete cumulative clean rounds +following its registry-documentation correction and a separate clean security +sitting. Step 6 (pure process/discovery substrate) is independently accepted after +two NEW complete cumulative clean rounds and a separate clean security sitting. +Step 7a (lazy runtime and finite compiled graph) is independently accepted after +two NEW complete cumulative clean rounds and a separate clean security sitting. +Step 7b and whole Step 7 are independently accepted at `c836ffe6` after two NEW +complete cumulative clean rounds, a clean security sitting and successful exact-tarball +Linux/macOS/Windows plus Node 22.0.0 consumers. Broader CI retains its separately +attributed status. Step 8a implements origin adapters and an inert native foundation; +it is independently accepted at `e0a5ae19` after two NEW complete cumulative +clean rounds and a separate clean security sitting. +Step 8b (creation-owned native processes) is independently accepted at +`e16a7659` after two NEW complete cumulative clean rounds and a separate clean +security sitting following its canonical-reference correction. Its unchanged +runtime/control evidence and successful full CI plus all four same-tarball +platform consumers remain attributed to `ab50be11`, with independently verified +source/dependency equivalence. Step 8c workflow/preparation/terminal integration +is independently accepted at `dbab25d8` after two NEW complete cumulative clean +rounds, a separate clean security sitting and successful full CI. All four +same-tarball platform consumers pass at that reviewed source. +**Step 8d and whole Step 8 are independently accepted on `development` at +`c83c8575` — 2026-10-17.** Two NEW complete cumulative CLEAN rounds (R25/R26) +each cover all 342 paths / 1,842 ordered hunks after the final fixture correction. +A NEW separate clean security sitting and all fifteen full-CI jobs pass at that +reviewed source, as do all four actual same-tarball platform consumers. Original +failed reviewer-preparation records and source/native/platform limits remain +explicit; source CI does not certify later documentation commits. The +[acceptance register](#wave-3-step-8d-and-whole-step-8-acceptance--2026-10-17) +joins exact evidence and historical corrections. Steps 1–8 are accepted; +**Steps 1–8 merged to `main` on 2026-10-18 as PR #95 (merge commit `6d2a7372`).** Wave 3 +itself is NOT complete: Steps 9–19 had not started at that merge (Step 9a.1, the first, is now +accepted on `development`, below, as are Steps 9a.2a and 9a.2b; the native-identity increment NI and Steps 9b.1 and 9b.2 are accepted too, and Steps 9c–19 have not started) and whole-wave acceptance remains Step 19. +The [merge checkpoint](#wave-3-steps-18-merge-checkpoint--2026-10-18) +records what merged and its review attributions. The final head `a3e164b5` (corrections from a +further independent review, including a secret-scrubbing rule) merged with author verification +only and was then independently reviewed (four rounds and a security sitting, with follow-up +fixes on `development`, not yet on `main`); CI for it and the merge commit finished green, and SonarCloud's `main` +gate (red since at least PR #90) was cleared on 2026-10-19. A pre-implementation review of Steps 9–19 +and the maintainer's [2026-10-19 rulings](#wave-3-steps-919-preflight-review-and-maintainer-rulings--2026-10-19) +give every open PR #95 finding an owner and record 29 increments (no new ADR). The PR95 increment +(the scrubber residuals, including both security residuals) is closed on `development` under the +maintainer's 2026-10-21 masker acceptance bar and round cap ([checkpoint](#wave-3-pr95-increment-implementation-checkpoint-and-review-round-corrections--2026-10-20)): rounds 1–10 were +NONCLEAN; the high and medium findings through round 9 are corrected, and after round 10, the last +round, only GEN-R10-1, which the maintainer counted as high on 2026-10-22, is corrected (in the summary, +by the author alone). The other findings are recorded in deferred-tasks.md. Step 9a.1 (the candidate build and finalizer, the first increment of the release map/custody substrate, #46/ADR-0111) is independently accepted on `development` at `cf97b7f1` (2026-10-24): after twelve NONCLEAN rounds and an internal rigor pass, its round 13 reported no blocker, high or medium finding and its security sitting was clean; the round's low findings are recorded in deferred-tasks.md ([acceptance](#wave-3-step-9a1-acceptance--2026-10-24)). Step 9a.2 is split into 9a.2a and 9a.2b ([preflight decisions](#wave-3-step-9a2-preflight-decisions--2026-10-24)); Step 9a.2a (the strict decoder, private manifest and packed-frame symbolication) is independently accepted on `development` at `a92a7602` (2026-10-24): rounds 1 and 2 were not accepted (a medium, then a regression from round 1's correction), and round 3 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](#wave-3-step-9a2a-acceptance--2026-10-24)). Step 9a.2b (the product map controls, the post-receipt rebuild refusals and the `package_test` receipt) is independently accepted on `development` at `0f73c253` (2026-10-24): round 1 was not accepted (a regression in the author's own deferral record and an untested exit code), and round 2 reported no blocker, high or medium finding and no regression; its low findings are recorded in deferred-tasks.md ([acceptance](#wave-3-step-9a2b-acceptance--2026-10-24)). The native-identity increment NI (a file-identity security fix the maintainer approved on 2026-10-24, register row NI, before Step 9b) is independently accepted on `development` at `fc5af701` (2026-10-25): round 1 reported no blocker, high or medium finding and no regression, with a clean security sitting; every product file and directory identity check now compares exact bigints, its low findings are recorded in deferred-tasks.md, and its Windows clause closes after three green advisory Windows runs ([acceptance](#wave-3-ni-acceptance--2026-10-25)). Step 9b is split into 9b.1 and 9b.2 ([preflight decisions](#wave-3-step-9b-preflight-decisions--2026-10-25)); Step 9b.1 (the private container, isolated GnuPG runner and encrypt-once library, synthetic recipients only) is independently accepted on `development` at `147c2788` (2026-10-25): rounds 1–3 were not accepted (one, one and two mediums, each corrected), and round 4 reported no blocker, high or medium finding and no regression, with a clean security sitting; its low findings are recorded in deferred-tasks.md ([acceptance](#wave-3-step-9b1-acceptance--2026-10-25)). Step 9b.2 (the release binding receipt v1, verified ciphertext reuse, the production recipient loader, absent in Wave 3, and scope separation; synthetic recipients only) is independently accepted on `development` at `563402e6` (2026-10-26): an internal rigor pass and rounds 1–3 were not accepted (mediums each time, all corrected), and round 4 reported no blocker, high or medium finding and no regression, its security sitting lows only; its low findings are recorded in deferred-tasks.md ([acceptance](#wave-3-step-9b2-acceptance--2026-10-26)). Step 9c is next. `development` is pushed after each accepted increment. No provider, +credential, billing or new ADR approval is pending for the approved implementation plan. diff --git a/docs/roadmap/phases/phase-2.6-conversational-authoring.md b/docs/roadmap/phases/phase-2.6-conversational-authoring.md index e8e69af1..c7f810ad 100644 --- a/docs/roadmap/phases/phase-2.6-conversational-authoring.md +++ b/docs/roadmap/phases/phase-2.6-conversational-authoring.md @@ -348,7 +348,8 @@ step's central change — had **zero test coverage** at any layer. All folded, e [deferred-tasks.md](../deferred-tasks.md). - **Reported spend is a systematic under-estimate of the provider's invoice** — an egress that streamed content but ended without a usage chunk, and a mid-stream failure, are recorded as 0 on *both* sides of the invariant. That is - a usage-capture gap in the seam, filed against **2.6.Q** (ADR-0070 §3). + a usage-capture gap in the seam, filed against **2.6.Q** (ADR-0070 §3) as its + [usage-capture item](#26q--dynamic-model-catalog-enrichment-modelsdev-per-model-capability-matrix--pricing-long-tail---p1p5-landed-pr-76-2026-07-26-p6-and-the-settings--models-toggle-open). - **Cross-turn tool-call memory** — a "the reseat forgets tool calls" report turned out to be an *engine* property, not a reseat one: tool pairs never cross **any** turn boundary (ADR-0062 §6). Deferred behind a default-off toggle, but **three already-live bugs gate it** (no pre-egress context-window check; overflow is a fatal, non-retryable @@ -1303,6 +1304,13 @@ models the four shipped adapters can **already** call and to ADR-0065 custom end opt-in *local-first*) and **clamp vs reject**. It closes the `error: validation` gap in [ADR-0066](../../decisions/0066-normalized-reasoning-effort-control.md). A **mandatory security review** of untrusted-community-pricing-into-a-safety-control is a co-gate. +- **Usage capture for streams that end without usage** *(added 2026-11-04, after the MVP; 2.6.C's open + obligation, [ADR-0070](../../decisions/0070-durable-per-model-session-cost-attribution.md) §3 exceptions + 1–2).* An egress that streamed content but ended without a usage chunk, and a mid-stream failure, are + recorded as 0 on both sides of the reconciliation invariant, so reported spend under-estimates the + provider's invoice. Capture or estimate their usage at the seam and mark it as such, or show the + under-report on the money surfaces; ADR-0070 §3's exceptions narrow with it. A PR #96 review found 2.6.C's + "filed against 2.6.Q" with no item here; the maintainer ruled this phase owns it. - **Tests:** an over-ceiling `max_tokens` is clamped per model; an OpenAI reasoning id uses `max_completion_tokens`; an out-of-range effort never reaches the wire and the picker offers only accepted tiers; a reachable-but-unpriced id gets a models-dev price so the cap applies (or is refused under @@ -1672,3 +1680,26 @@ flowchart LR | Competitor-copy erodes Relavium's posture | Patterns are stolen, postures are not: keychain-only secrets, fail-closed approval, YAML-first artifacts, and the ADR-0049 machine contract are non-negotiable filters on every borrowed idea | Part of [roadmap/](../README.md). Carry-over hardening lives in [../deferred-tasks.md](../deferred-tasks.md). +## Status record moved from current.md — 2026-11-04 + +> Moved verbatim from `docs/roadmap/current.md` as it stood at `5a328bca` by the +> [PR #96 review correction](phase-2.5.5-hardening-and-remediation.md#moved-by-the-pr-96-review-correction--2026-11-04). +> Only relative links were rewritten for this directory and heading levels lowered by one. + +### Not started yet + +The rest of **Phase 2.6 — Conversational Authoring and the First-Class CLI** +([phase-2.6-conversational-authoring.md](), in progress — +**2.6.F is done**, +unblocked by the 2.5 close and **re-scoped 2026-07-08** from maintainer UX findings + a competitor +research pass + the deferred-tasks triage): a full-screen, Home-managed CLI (browsers for +workflows/runs/agents, provider + MCP + settings management, onboarding v2 with the Relavium-account +stub), competitor-breadth tools under the governance floor, settings/theming/`en`+`tr` localization, +the run-ops resume follow-up — plus the original spine: the shared `@relavium/authoring` package and +a chat that **authors** standards-valid `.relavium.yaml`. Then the surfaces and the cloud — +everything after the engine critical path: the desktop app (Phase 3) and the VS Code extension (Phase 4), then **Product Phase 2** — first **managed +inference** ([phase-5-managed-inference.md](phase-5-managed-inference.md), the opt-in +`managed` gateway, engine still local), then the **cloud execution layer and web portal** +([phase-6-cloud-execution-portal.md](phase-6-cloud-execution-portal.md)), the two +decoupled per Option B. See the [phase index](../README.md#phase-index) and the +[milestone spine](../README.md#global-milestone-spine) (M3 onward). diff --git a/docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md b/docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md index 1fad5b12..b3c1bbd3 100644 --- a/docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md +++ b/docs/roadmap/phases/phase-2.6.5-core-reliability-remediation.md @@ -7090,3 +7090,603 @@ blocker, followed by canonical request conformance and unknown usage. Its prefli ADR-0087's accepted corrections with the current W7 lifetime contracts. After W8, re-evaluate every phase exit criterion before the next remediation wave opens; [current.md](../current.md#next-work--wave-2--shut-the-doors) remains canonical for execution order. W8 implementation is not started by this documentation update. +## Execution record moved from current.md — 2026-11-04 + +> Moved verbatim from `docs/roadmap/current.md` as it stood at `5a328bca` by the +> [PR #96 review correction](phase-2.5.5-hardening-and-remediation.md#moved-by-the-pr-96-review-correction--2026-11-04): +> the D66 reset had kept only current.md's live status and order and recorded its 2026-07-26 plan in the Phase 2.5.5 +> register, and these Phase 2.6.5 records had left the docs. Only relative links were rewritten for this directory +> and heading levels lowered by one; a heading in brackets is added. The `W4` start-gate paragraph is kept in +> [its earlier section](#w4-start-gate--maintainer-ruling-moved-from-currentmd-on-2026-11-04). + +### [current.md's header notes, as of 2026-10-25] + + + +> **W7 merged to `main`, 2026-10-10 (PR #90, `46a6a701`):** all twelve steps, five genuine overflow records, +> production request ownership and consolidated host lifetimes have independent acceptance. +> All six W7 items are closed; Phase 2.6.5 stood at **47/51 at that merge**. The +> [closing register](#w7-closing-register--2026-10-10) +> joins per-item causal controls, both final review rounds and every submitted systematic finding. +> PR #90 merged at 13:57:47 UTC. W8 was still open at that checkpoint; it subsequently merged in PR #92. Approved deferrals remain. +> No implementation, user decision, credential, billing or provider capture is pending for W7. + +> **Post-closure corrections accepted, 2026-10-10:** the [new correction register](#post-closure-systematic-review--2026-10-10) +> joins every new report disposition. Core, CLI/DB and tooling/docs each have two NEW complete clean +> cumulative review rounds. Fresh unchanged-production coverage passes 447 files / 9,512 cases / +> eleven existing capability/key-gated LLM skips. Published-head checks are reported in PR #90. +> W7 has no implementation, provider, credential, billing or user-decision prerequisite left. +> The corrections merged with PR #90; W8 subsequently merged in PR #92. Existing approved limits remain. + +> **Publication tool corrections accepted, 2026-10-10:** [two NEW complete clean cumulative rounds](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-10T13-35-14-w7-publication-sonar-round-2-review.md) +> independently verify all four Sonar-label dispositions, preserved UTF-16 ordering and OS-owned Git +> dispatch with PATH-trap causal controls. Shipping production/tests remain unchanged. Final local CI +> and exact published-head GitHub/Sonar checks are reported in PR #90; the prior failed analysis stays recorded. +> W7 has no implementation, capture, credential, billing or user-decision prerequisite left. + + +### W8 merged — Phase 2.6.5 complete + +**W8 merged to `main` on 2026-10-11 as [PR #92](https://github.com/HodeTech/Relavium/pull/92), +merge commit `ef0ffa54`. Phase 2.6.5 is complete with approved deferrals: 49/51 items closed.** +The [merge checkpoint](#w8-merge-checkpoint--2026-10-11) +joins final publication evidence and all seven phase exit criteria. All seven W8 steps, +CR-81, CR-82 and ADR-0087 §1 are complete on `main`; no W8 implementation, review, +publication, provider, billing, credential or user-decision gate remains. + +1. **Consumption:** explicit run/session modes preserve iterated delivery from construction; + subscribe-only handles retain no primary buffer and reject iteration with a typed error. +2. **Conformance:** strict expected exchanges and the 24-case actual installed-SDK HTTP + matrix detect production-lowering regressions. +3. **Usage uncertainty:** complete/partial/missing observations, pending ACK before invocation, + once-only settlement and late receipts, sticky old-writer uncertainty and qualified + human/JSON totals preserve the distinction between unknown usage and actual zero. + +The [original closing register](#w8-closing-register--2026-10-11) +preserves its source acceptance and two NEW complete whole-wave reviews. The +[maintainer correction register](#w8-pr-92-systematic-review-corrections--2026-10-11) +joins all report dispositions and two NEW independent complete clean cumulative rounds per +correction increment. Final reviewed head `3d4491df` passed both seven-job PR/push CI +workflows and the SonarCloud quality gate. Published ordinary/enforced-coverage inventories +are 511 files / 11,026 passing tests / 12 established skips; four wire and 23 usage +mechanisms retain 427 permanent cases and 69 joined native children. Fresh full local +CI and bare/enforced coverage at `6139563e` remain separately attributed evidence. +No merge-commit execution, live-provider invoice or unsupported-platform result is inferred. +Two nonblocking Sonar complexity suggestions remain visible. Detailed review records stay +local and ignored; historical citations remain commit-pinned. + +**What remains:** CR-70 (cross-turn tool-history carrying) and CR-93 (tenant-scoped mutable +state) are approved deferrals. The 49/51 count includes CR-95's short-term fail-closed fix; +its long continuation and existing exact-capacity/scoped residuals remain in +[deferred-tasks.md](../deferred-tasks.md), with their existing severity, trigger and narrowed +claim. ADR-0087 §1 is outside the denominator. Phase closure does not claim 51 fully +implemented items or an empty debt backlog. Phase 2.5.5 and Phase 2.6 remain in progress. +Earlier dated pending checkpoints in this page and the phase register are historical. + + + + + + +### Historical W7 implementation checkpoints + +Dated W7 checkpoints throughout this page retain their original pending states. The closing checkpoint above +and its later post-closure and merge checkpoints control current W7 status; historical pending/draft +wording describes the checkpoint when it was written. + +> **Prior complete composition review:** [host-closure round 5](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-10T00-38-24-w7-host-closure-round-5-review.md) +> finds no new confirmed actionable issue across all 126 paths / 364 hunks. This is the first +> clean complete cumulative round; one further fresh complete clean round remains required. +> Step 8 and final Step 12 remain open, with no user/provider blocker. +> +> **Complete composition review:** [host-closure round 4](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-10T00-22-00-w7-host-closure-round-4-review.md) +> confirms public stream exit can skip underlying cleanup on retirement failure. Parent adds +> twelve paired permanent cases and guarantees cleanup while preserving the original throw. +> Two fresh complete clean rounds, Step 8 and final Step 12 remain required; no user input is pending. +> +> **Complete composition review:** [host-closure round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T23-56-41-w7-host-closure-round-3-review.md) +> confirms two retirement-boundary Highs: mutable returned output/usage and failed cleanup +> suppressing accounting/acknowledgement. Parent corrections add thirty-two permanent cases. +> Fresh complete review remains required; Step 8/12 remain open, with no user/provider blocker. +> +> **Complete composition review:** [host-closure round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T23-34-00-w7-host-closure-round-2-review.md) +> confirms due-budget departure starvation and provider entry after synchronous cancellation. +> Parent corrections add nineteen permanent cases and fix adjacent canonical-documentation drift. +> Fresh complete review is required; Step 8/12 remain open. No user/provider blocker exists. +> +> **Complete composition review:** [host-closure round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T23-05-00-w7-host-closure-round-1-review.md) +> confirms three lifecycle High findings (post-pause receipt cursor, rejected activation joining, +> post-unmount Ink visibility) and a required shipping-registry acceptance gap. Parent corrections +> add eighteen permanent cases and strengthen native output assertions. Fresh complete review of +> the corrected composition remains required before acceptance; Step 8/12 are still open. + +> **W7 live status, 2026-10-09:** all five immutable provider records and Step 7's four-round +> acceptance remain complete. [Two fresh adapter/chain round-3 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-08T23-47-31-w7-owned-adapter-chain-round-3-review.md) +> accept all 24 cumulative paths/78 hunks through `0c3c185b`; exact-head required CI, coverage, +> Node 22, Windows, peers and Sonar pass. Core measured-round reuse is implemented at `99e78e55`; +> [both independent round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T00-16-47-w7-owned-core-round-1-review.md) +> clear all 13 paths/47 hunks. [Two fresh round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T01-00-00-w7-owned-core-round-2-review.md) +> accept all 19 cumulative paths/53 hunks through `f1281214`; exact-head CI passes. ADR-0103 +> append/receipt integration is implemented at `a35739a4`; +> [both complete round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T01-27-00-w7-append-receipt-round-1-review.md) clear all 17 paths/57 hunks. +> [Two fresh cumulative round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T02-00-00-w7-append-receipt-round-2-review.md) accept all 23 paths/63 hunks +> through `01115050`. Raw provider/iterator/media and native child lifetime integration is +> implemented at `fde3ca0c`; [both complete round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T02-36-10-w7-raw-producer-round-1-review.md) +> verify a public-return High, corrected with two regressions (28 permanent cases total). +> [Two fresh cumulative round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T03-25-00-w7-raw-producer-round-2-review.md) accept all 20 paths/41 hunks +> through `1172d247` after complete report/artifact/freeze audits; nested Corepack reviewer +> violations get no proof credit. The HTTP descendant increment now has 23 new permanent +> cases, 16 compiled negative controls and a passing cache-qualified root CI; +> [both complete round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T04-16-24-w7-http-descendants-round-1-review.md) +> clear all 22 paths/38 hunks through `db02b83c` after Parent's complete artifact/freeze audits. +> [Two fresh cumulative round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T04-55-00-w7-http-descendants-round-2-review.md) +> accept all 24 paths/40 hunks through `6e620556` after Parent's complete evidence and 645,606-entry freeze audits. +> Invocation-local provider/raw-poll integration adds 57 permanent cases; [complete round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T05-43-00-w7-provider-invocation-round-1-review.md) +> verify a method-receiver defect and separately strengthen a weak Promise test. +> [Both complete round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T06-33-00-w7-provider-invocation-round-2-review.md) verify one getter-acquisition defect. +> Its correction passes root CI (404 files / 9,030 cases / 12 existing skips); +> [both NEW complete cumulative round-3 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T07-06-28-w7-provider-invocation-round-3-review.md) +> and Parent's complete artifact/645,743-entry freeze audits accept all 28 paths/67 hunks through +> `f9a8ff37`, with no new confirmed finding. Exact-head required CI, coverage and Sonar pass. +> MCP transport/handler descendants now have 77 new permanent cases. Rounds 1–3 corrected +> queued-reservation, deadline, disposer, coverage, empty-start stdio and callback failures. +> [Complete round-4 reviews and Parent evidence/freeze audits](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T12-58-01-w7-mcp-lifetime-round-4-review.md) verify +> completed control-lane reference retention. Correction `37220a95` releases only fully +> acknowledged lanes and passes actual root CI; seven new causal regressions and fresh enforced +> private coverage pass (412 files / 9,107 cases / 12 skips; MCP branches 90.68%). +> [Two NEW complete cumulative round-5 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T13-37-00-w7-mcp-lifetime-round-5-review.md) +> and Parent's complete source/artifact audits accept all 38 paths/70 hunks through `99c46bbd`, +> with no new confirmed finding and 268 independently passing cases per reviewer. The full +> shared-tree audit has zero file/link changes and one qualified `.git` directory mtime change. +> Fresh startup registers its complete continuation. [Two complete first-round reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T14-28-09-w7-startup-lifetime-round-1-review.md) +> and Parent audits verify reentrant clock/timer cancellation; three permanent regressions +> cover the correction. [Two NEW complete cumulative round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T14-55-12-w7-startup-lifetime-round-2-review.md) +> and Parent's exact-source/physical-artifact/647,041-entry freeze audits accept all ten +> paths/sixteen hunks through `5d44b13b`, with zero new confirmed findings and 2,850 core cases per reviewer. +> [Both complete scheduler readiness round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T15-44-15-w7-scheduler-readiness-round-1-review.md) +> verify a secondary readiness-diagnostic failure. Its correction has twelve causal readiness +> regressions and 2,862 passing private core cases. [Two NEW complete cumulative round-2 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T16-14-29-w7-scheduler-readiness-round-2-review.md) +> and Parent's source/artifact/647,245-entry unchanged freeze audit accept all nine paths/ten +> hunks through `95f3923a`, with zero new readiness findings. Both inherited dispatch Highs +> (post-start-append cancellation admission and unsafe detached failure formatting) are +> corrected in `1918690d`, with six new permanent cases and passing exact-input root CI. +> [Both complete dispatch-correction round-1 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T17-28-06-w7-dispatch-corrections-round-1-review.md) +> and Parent's artifact/647,355-entry unchanged freeze audits accept the eight-path/ +> seventeen-hunk scope with zero confirmed findings. [Both complete cumulative second-round reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T17-58-41-w7-dispatch-corrections-round-2-review.md) +> verify an inherited retry-start cancellation High. Its correction and a subsequent +> Parent-discovered synchronous attempt-entry correction add ten permanent cases; +> final-source causal controls pass. NEW complete cumulative reviews are required +> before scoped closure; complete scheduler/dispatch roots stay open. +> [Complete cumulative round-3 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T18-37-30-w7-dispatch-corrections-round-3-review.md) verify a retry/deadline duplicate-terminal High. +> Parent confirms and corrects it, with a new permanent causal regression; fresh cumulative review remains required. +> [Complete cumulative round-4 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T19-10-39-w7-dispatch-corrections-round-4-review.md) verify two paths of one attempt-correlation High. +> Parent corrects explicit failure forwarding and first-start reset with five permanent cases; NEW cumulative review is required. +> [Complete cumulative round-5 reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T19-41-03-w7-dispatch-corrections-round-5-review.md) verify a pre-start readiness attempt-correlation High. +> Parent corrects its latest-entered-attempt forwarding with two paired permanent cases; NEW cumulative review is required. +> [Complete cumulative round 6](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T20-08-19-w7-dispatch-corrections-round-6-review.md) finds no new confirmed issue. +> [The NEW complete round 7](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-09T20-29-02-w7-dispatch-corrections-round-7-review.md) verifies a differing-order grace High: +> an unstarted claim can receive a failure. Parent corrects explicit claim/start tracking, +> with three permanent causal cases; NEW complete cumulative reviews remain required. +> A broader private actor prototype exposed CLI closure/lease-handoff failures in root CI. +> Complete scheduler/dispatch/resume registration must land with shipping host departure; +> original failures and assertions are retained, and that integration stays required in W7. +> Complete MCP startup/all-actor retirement, custom-provider descendants and remaining host departure work +> and Step 8 atomic compaction/recovery remain open. The ownership/lifecycle +> Highs, Step 12, final whole-wave CI/coverage/Sonar and all six W7 register items remain open. +> Draft PR #90 is unmerged; no further paid call, credential or ADR approval is needed. +> Earlier progress statements below describe their publication checkpoints. + +> **W7 implementation, 2026-10-02:** the documentation review is closed. The maintainer approved +> [ADR-0099](../../decisions/0099-compaction-has-an-idle-budget-outcome-and-an-unknown-window-policy.md), +> [ADR-0100](../../decisions/0100-budget-authorization-is-durable-state-with-a-replay-barrier.md) and the +> [twelve-step W7 plan](#w7-pre-implementation-review-and-proposed-execution-plan--2026-10-02). +> Work remains on `development`; each step closes with two fresh independent Codex review rounds and verified +> fixes. Step 1's implementation and first review corrections pass `pnpm run ci` and `pnpm coverage`; +> [round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T03-04-05-w7-step-1-round-1-review.md) verified seven findings; +> [round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T03-22-32-w7-step-1-round-2-review.md) verified a cleanup race in its fix. +> [Round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T03-35-41-w7-step-1-round-3-review.md) verified a passing/starting race; +> cleanup now retains shared parents and deletes only owned probes. All corrections pass the gates; +> [Round 4](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T03-43-31-w7-step-1-round-4-review.md) cleared probe ownership and found +> a capture pathname-cleanup race. The runner now closes its descriptor without deleting a failed +> destination. [Round 5](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T03-50-35-w7-step-1-round-5-review.md) cleared the +> complete step and final corrections; step 1 is closed. Step 2's structural store boundary and +> durable effect-turn allocator pass the required checks and root coverage (289 suites, 6,367 passing +> tests, 11 skipped); full CI passed at `1927daa1`. +> [Round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T04-29-12-w7-step-2-round-1-review.md) verified nested-transaction +> key reuse and unsafe session media metadata. Both are corrected; checks and coverage pass +> (289 suites, 6,374 passing tests, 11 skipped), with full CI passing at `c96eb9b5`. +> [Round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T04-41-54-w7-step-2-round-2-review.md) cleared the complete step +> and both fixes, including real lock contention and process-crash checks. Step 2 is closed; +> step 3 implementation now connects structural completed turns, atomic persistence, empty-final +> resume/export and per-call effect ids across the CLI hosts. Required checks and coverage pass; +> [Round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T05-34-44-w7-step-3-round-1-review.md) verified a deep-JSON +> metadata failure and a lost policy-denial classification. Both corrections pass the required +> checks and coverage (291 suites, 6,406 passing tests, 11 skipped); full CI passes at `4045272c`. +> [Round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T05-48-54-w7-step-3-round-2-review.md) verified a cached-command +> durability bypass and a missing tool name on recovered filesystem denial. Both corrections pass +> checks and coverage (291 suites, 6,410 passing tests, 11 skipped); full CI passes at `e95a6010`. +> [Round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T06-10-38-w7-step-3-round-3-review.md) independently verified an +> unjournaled-tool durability bypass and a stale export introduction. Dispatch admission now checks +> the live latch after approval/preparation for every actual tool call and releases a proven +> non-dispatch claim. Both corrections pass checks and coverage (291 suites, 6,415 passing tests, +> 11 skipped); full CI passes at `dd50738c`. +> [Round 4](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T06-34-39-w7-step-3-round-4-review.md) cleared dispatch admission +> and verified loss of nontrailing pre-W7 empty-final user text from resume/reseat and export prompts. +> Legacy context and boundary slots now preserve the old read behaviour without fabricating terminals +> or completed counts. Required checks and coverage pass (291 suites, 6,422 passing tests, +> 11 skipped); full CI passes at `6d5b3f0b`. +> [Round 5](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T06-48-45-w7-step-3-round-5-review.md) cleared the complete step +> and all earlier corrections, including independent mixed-history projection, YAML, real SQLite +> boundary and process-crash probes. **Step 3 is closed; step 4 proceeds automatically** with +> session-result privacy, full-history disclosure snapshots and exact committed-row retention. +> Step 4's [first acceptance round](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T08-05-02-w7-step-4-round-1-review.md) +> independently reproduced pre-mount Ink resume/reseat deletion. The verified correction separates +> opening from committed-mount activation and carries driver activity through disclosure and teardown; +> [Round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T08-48-12-w7-step-4-round-2-review.md) reproduced activation before passive Ink input setup +> and before disclosure was drawn. Activation now awaits usable setup and disclosure render acknowledgement, +> observes renderer exit/error and keeps input gated; actual Ink and Home tests verify the display boundary. +> Required checks and coverage pass (297 suites, 6,487 passing tests, 11 skipped); a third fresh +> corrective acceptance round was required before step 4 closes. +> [Round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T09-24-27-w7-step-4-round-3-review.md) verified incomplete-call attribution and +> flush acknowledgement of unwritable output. Completion now uses the shared structural projector; +> rendered disclosure checks the actual Ink output and its lifetime. Required checks, build and coverage +> pass (297 suites, 6,497 passing tests, 11 skipped); a fourth fresh acceptance round is required. +> [Round 4](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T10-04-04-w7-step-4-round-4-review.md) verified headless stderr enqueueing mistaken for delivery +> and inline disclosure lost during Ctrl-Z suspension. Native stderr callbacks now acknowledge delivery; +> notice publication and render acknowledgement share terminal ownership with suspension. Nine focused suites +> pass 217 tests, including delayed native delivery and independent real STOP/CONT controls. +> Required workspace checks, build and coverage pass (298 suites, 6,514 tests, 11 skipped). Full CI +> passed at `b9d52d76`; round 5 required two more corrections. +> [Round 5](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T10-36-22-w7-step-4-round-5-review.md) independently reproduced blocked native stderr +> holding Ctrl-C teardown and late Home acknowledgement deleting evidence during canceled-session MCP close. +> Headless activation now races an owned interrupt; Home activation requires a running session outside teardown. +> Eight focused suites pass 310 tests; native pipe/FIFO before/after controls and late callbacks after actual +> database close pass. Workspace checks, build and coverage pass (298 suites, 6,520 tests, 11 skipped). +> Full CI passed at `e753beb7`. [Round 6](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T10-57-58-w7-step-4-round-6-review.md) accepted the complete +> step and every correction with no new material findings. Parent native/privacy controls pass; compiled-entry +> process-exit delay under queued stdio also reproduces on the exact pre-step baseline and is explicitly qualified. +> **Step 4 is closed; step 5 proceeds automatically.** All six W7 register items remain open. +> Step 5's memory implementation and canonical docs are checked: +> one pure request projection, frozen policy, completed-turn spans and engine/CLI history refusals; +> actual request, workflow-node and SQLite resume/reseat controls pass. Workspace checks, build and coverage +> pass (299 suites, 6,567 passing tests, 11 skipped). Later measured pre-send/recovery and revised compaction +> outcomes remain staged. Clean-tree full CI passed at `ecdcc395`. +> [Step 5 round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T12-00-17-w7-step-5-round-1-review.md) accepted the implementation: +> both reviewers passed 347 focused tests; parent-rerun controls passed 390 state/history cases and 86 +> actual CLI/SQLite cases. Matched source-head operational controls pass; exact pre-step engine controls +> fail as expected. Adjacent working-context/durable-history wording is corrected. +> [Step 5 round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T12-21-33-w7-step-5-round-2-review.md) accepted the complete step: +> both fresh reviewers passed 347 tests; the parent reproduced 1,536 state/history and 86 CLI/Home/SQLite +> cases, plus exact-source positive/negative controls. **Steps 1–5 are closed; all six W7 items remain open.** +> Step 6's independently reviewed [Accepted ADR-0101](../../decisions/0101-configured-output-estimates-apply-only-when-the-wire-is-uncapped.md) +> resolves a newly identified configured-estimate/native-cap precedence gap. The maintainer approved the +> complete revised decision on 2026-10-02; dated forward notes are landed and step 6 proceeds; +> [ADR revision round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T15-09-58-w7-adr-0101-revision-round-1-review.md) records the maintainer's +> verified transport/pricing findings and the corrected measurement-to-attempt handoff. +> [Fresh draft round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T15-18-01-w7-adr-0101-revision-round-2-review.md) accepted the complete +> corrected proposal with no actionable findings; full CI passed at `2b628e1a`. The acceptance and runtime +> obligations remain distinct: runtime acceptance is now complete. +> [Step 6 round 8](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T22-44-01-w7-step-6-round-8-review.md) accepts the complete step and seven corrections. +> Both fresh reviewers reported no verified new findings; parent replays pass 120 contract and 54 +> financial controls. **Steps 1–6 are closed; all six W7 register items remain open.** +> Step 9's quote/debit foundation is implemented, with [candidate preflight and parent causal controls](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-02T23-40-16-w7-step-9-foundation-preflight-review.md). +> Full CI/coverage pass after the round 3 correction (315 test files, 7,152 passing tests, 11 existing skips). +> [Round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T00-25-08-w7-step-9-round-1-review.md) verified and corrected two caller settlement findings; [Round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T01-10-45-w7-step-9-round-2-review.md) verified and corrected the async consumer and cancellation teardown; [Round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T01-57-14-w7-step-9-round-3-review.md) verified and corrected park/poll host admission loss and the reentrant durable-total race. +> [Fresh round 4](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T02-40-53-w7-step-9-round-4-review.md) accepts the complete foundation with no new findings; both sealed inventories and 70 parent-rerun fresh cases are verified. Its corrected native-loader evidence and prior harness isolation exception are recorded explicitly. +> **Step 9 is closed; approved Step 10 proceeds automatically** with durable workflow activation and replay. +> [Step 10 round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T08-35-49-w7-step-10-round-1-review.md) reproduced and corrected +> passive-resume lease expiry and unbound predecessor CJS/peer/optional edges. Exact-fence renewal and +> an owned, versioned dependency closure pass causal controls, full CI and coverage +> (320 suites, 7,318 passing tests, 11 existing skips), including seventy actual-runner cases, +> twelve real predecessor refusals, two legacy controls and seven permanent graph/runtime controls. +> [Step 10 round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T11-47-48-w7-step-10-round-2-review.md) verified and corrected +> terminal conservative-money ordering, historical budget input compatibility, companion amount symmetry +> and pinned runtime edges. Full CI/coverage pass (320 suites, 7,340 passing tests, 11 existing skips); +> 78 native runner cases, thirteen graph controls and actual historical-input replay pass causal checks. +> [Step 10 round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T12-39-00-w7-step-10-round-3-review.md) verifies and corrects +> authority-witnessed optional-amount duplicate compatibility, preserving strict money conflicts, +> genuine output and unrelated pending work. Eighteen permanent regressions and 300 native selected +> cases pass. Full CI/coverage pass (320 suites, 7,358 tests, 11 existing skips); fresh full-Step-10 +> round 4 precedes closure. All six register items remain open. +> [Step 10 round 4](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T13-33-00-w7-step-10-round-4-review.md) corrects lease cleanup rejection replacing +> primary typed refusals or an already-terminal no-op. Eighteen native regressions cover all six +> guards and successor takeover; full CI/coverage pass (320 suites, 7,376 tests, 11 existing skips). +> Fresh whole-Step-10 round 5 precedes closure; no residual is deferred. +> [Step 10 round 5](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T14-31-39-w7-step-10-round-5-review.md) corrects inherited parallel gate publication handing off +> and then reclaiming a waiting-only lease. Ten permanent regressions cover held pause ACKs, +> actual SQLite fresh-owner resume and immediate human-decision media preparation. Async final-ACK +> limits are explicit; no new clock/timeout policy is chosen. Full CI/coverage pass (320 suites, +> 7,386 tests, 11 existing skips). Fresh whole-Step-10 round 6 precedes closure. +> [Step 10 round 6](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T15-42-00-w7-step-10-round-6-review.md) corrects inherited ordinary +> decision media completion after cancellation, deadline or fenced closure. Twelve permanent +> Core/actual SQLite-CAS regressions prevent late subscriber success and renewed run references; +> guard removal/restoration confirms the failure. Full CI/coverage pass (321 suites, 7,398 tests, +> 11 existing skips). Fresh full-Step-10 round 7 precedes closure. +> [Step 10 round 7](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T16-54-28-w7-step-10-round-7-review.md) corrects inherited media references written before a stale +> event fence is refused. Produced handles now follow append acknowledgement; owned uncertain-terminal +> media remains retained for the outbox. Seven permanent successor/terminal controls and independently +> verified sealed evidence pass, with full CI and sequential coverage (323 files, 7,405 tests, +> 11 existing skips). Fresh full-Step-10 round 8 precedes closure. +> [Step 10 round 8](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T17-50-23-w7-step-10-round-8-review.md) accepts the full 74-file step with no new verified findings. +> Both fresh reviewers pass 857 permanent tests and separate five/six fresh probes; complete reports and +> every sealed inventory entry are parent-verified. **Step 10 is closed; approved Step 11 proceeds automatically** +> with the safe CLI surface, permanent compiled-process smoke coverage and fresh independent reviews. +> Step 11 implementation now passes full CI and sequential coverage (327 files, 7,506 tests, 11 existing skips), +> including permanent compiled CLI refusal/status/abort/MCP-interruption smokes and raw-triple display regressions. +> [Step 11 round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T19-39-26-w7-step-11-round-1-review.md) independently verifies native approval-write cancellation, +> the no-MCP signal guard and stale identified-companion display cleanup. The parent fixes all three, +> adds passive Core cancellation/fence controls and corrects the ownership wording. +> Full CI and sequential coverage pass after the corrections (327 files, 7,519 passing tests, +> 11 existing skips; global line/branch/function 95.88%/92.63%/96.73%). +> [Step 11 round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-03T21-01-16-w7-step-11-round-2-review.md) verifies two further High findings: +> a successful native busy wait can conceal OS SIGINT until after egress, and terminal write uncertainty +> was misreported as takeover. The CLI host yields across a poll turn before decision acknowledgement; +> terminal uncertainty retains exit 5, while actual successor fencing still refuses without a false +> claim that an earlier approval is absent. Seven new permanent controls pass; both complete sealed +> reports and every inventory entry are parent-verified. Full CI and sequential coverage pass +> (327 files, 7,526 tests, 11 existing skips; 95.88%/92.63%/96.73%). Fresh full-Step-11 round 3 follows +> the correction commits; the step remains open. +> [Fresh Step 11 round 3](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T01-17-23-w7-step-11-round-3-review.md) is clean in both +> complete independent reviews: 452 permanent tests each plus separately reported fresh native, +> passive and compiled-process controls. Both reports and every sealed inventory entry are verified. +> Interrupted earlier round-3 runs contribute no acceptance. **Step 11 is closed.** +> [Independent Step 12 document preflight](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T01-04-15-w7-step-12-document-preflight-review.md) +> verifies and corrects usage/pricing and fixture-coverage prose; connected at-rest/event claims, +> guides and moved follow-up status are reconciled. This independent maintenance proceeds through +> repository checks and fresh post-commit review; Step 12's whole-wave acceptance remains open. +> [Committed maintenance round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T07-00-17-w7-step-12-maintenance-round-1-review.md) +> independently verifies one Medium reservation-existence qualification and one Low inherited +> event-table guarantee. Both are parent-verified and corrected in four text/comment locations. +> Complete reports and all 2,687 sealed entries are verified; 81 fresh reviewer DB tests and +> a real-writer reference-corruption/restoration control pass. Fresh complete maintenance +> round 2 follows the correction commits; whole Step 12 remains open. +> [Committed maintenance round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T07-25-20-w7-step-12-maintenance-round-2-review.md) is clean +> across all 26 paths/37 hunks in both fresh reviews. Both complete reports and all 2,574 sealed +> entries/source pins are verified; fresh 81-test DB/strict checks and real-writer causal controls pass. +> **The independent maintenance subset is closed; whole Step 12 and all six W7 items remain open.** +> [Draft PR #90](https://github.com/HodeTech/Relavium/pull/90) is open on `development`. +> Required Linux CI revealed an actual-Ink test-context mismatch. Further controlled checks found +> undisplayed disclosure under the existing CI opt-outs and Home's inline reseat Static cursor; +> corrections pass two fresh independent review rounds. Prior scoped acceptance does not waive other findings. +> All five required actual overflow records (Anthropic input rejection and native context stop, +> OpenAI, DeepSeek and Gemini) are now preserved in the +> [fixture index](../../../packages/llm/src/conformance/fixtures/overflow/README.md). +> The separate Anthropic native context-stop response was captured on 2026-10-08; +> the evidence prerequisite is satisfied. Step 7 classification, Step 8 and whole-wave acceptance remain open. +> Step 1 repaired collection without deleting private artifacts and provides the +> [maintainer-run live overflow capture procedure](../../runbooks/capture-provider-overflow.md). +> Classification must replay the preserved live fixtures. The six W7 items stay open. + +> **PR 90 systematic follow-up — 2026-10-04.** Two maintainer reports against `95137c34` register 35 +> deduplicated findings, with [qualified independent triage](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T10-35-00-w7-systematic-review-intake-review.md). +> Step 6's request-ownership guarantee is reopened; adjacent session/CLI, replay-tool and documentation +> obligations require correction and fresh acceptance. The [first committed rendering review](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T10-32-37-w7-rendering-round-1-review.md) +> reproduces notice loss after a supported chat resizes and CI route/ownership drift. Its corrections +> are accepted by [fresh round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T12-00-00-w7-rendering-round-2-review.md). +> The rendering corrective group is closed. Previous step closure records remain historical evidence, +> not current whole-wave acceptance. All ten GitHub workflow checks pass at `d4e4fa9c`; SonarCloud and remaining findings stay +> separately under examination. No finding is deferred at intake. Steps 7–8, whole Step 12 and all six +> W7 register items remain open. + +> **Request-ownership decision preflight — 2026-10-04.** [Proposed ADR-0102](../../decisions/0102-a-measured-request-owns-its-inert-data-through-egress.md) +> would close the measured-to-sent alias gap through one owned inert construction and existing candidate +> cap authority. It deliberately narrows non-cap opaque inputs and requires maintainer approval before +> dependent implementation. [Draft round 1](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T12-21-41-w7-adr-0102-round-1-review.md) +> corrects SDK shared-reference parity and incoming measured-plan ordering. [Fresh complete round 2](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T13-02-54-w7-adr-0102-round-2-review.md) +> receives no new material proposal findings; maintainer approval remains required before dependent +> implementation. Independent corrective work continues. The ownership HIGH remains open. + +> **Systematic group 2a, round 1 — 2026-10-04.** [Two complete independent reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T13-47-56-w7-systematic-group-2a-round-1-review.md) +> verify the cap/structural repairs but reproduce a new successful-tool observer regression: +> real session usage is lost and the hard turn cap can be bypassed. Corrections retain classified +> engagement/usage and fix the normalizer comment. Forced root checks and build pass; **fresh +> complete round 2 precedes group closure**. Ownership, remaining groups and W7 acceptance stay open. + +> **Systematic group 2a, round 2 — 2026-10-04.** [Both complete reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T14-41-03-w7-systematic-group-2a-round-2-review.md) +> verify the successful observer correction but reproduce an inherited failed-tool observer High. +> `41a3ef9a` retains real usage/engagement for invalid-argument and read-failure notifications, +> including EA7 abort, without broadening registry recovery. Eight new regressions fail before +> repair; forced root checks/build pass afterward. **Fresh complete round 3 precedes closure.** +> Ownership, other groups and whole W7 acceptance remain open. + + +### [The Phase 2.6.5 interlude in current.md's execution order] + +#### Phase 2.6.5 — Core reliability remediation (between Wave 1 and Wave 2) + +Three independent core reviews of the post-Wave-1 tree converged — independently — on the **same seven blocking +gaps**: effect journal, stdio MCP consent-before-spawn, run lease, compaction trust elevation, stream grammar, +input admission, event-log ordering. Three separate reviews landing on the same seven points is not opinion. + +The full, self-contained work list is +[phase-2.6.5-core-reliability-remediation.md]() — **51 items** (`CR-01`…`CR-98`; `CR-96` and `CR-97` were opened 2026-09-14 from the `W7` ADR reviews, `CR-98` on 2026-09-18) with evidence, fix, acceptance criteria and a decision/ADR/gate register, written so the work +can be done from that document alone. An adversarial plan review on 2026-08-10 corrected the phase boundary, +the exit rule and the execution order, and added two items (`CR-17` resume identity, `CR-63` `input_schema` +docs-only); `CR-64` came from the Batch 1 triage, `CR-21b` from ADR-0082 §10 and `CR-21c` from the `W2` +document review on 2026-08-25, which is why the total has moved since the list was first written. + +> **Historical phase checkpoint — 41 of 51 closed (updated 2026-09-18; at W6's merge it was 39 of 48 — see the register for why not 43: W5 was counted twice). `W0`–`W2` merged clean; `W3` merged 2026-08-30 (PR #86) with one reproduced BLOCKER and nine verified findings open — see the `W3` residuals in [deferred-tasks.md](../deferred-tasks.md). `W4` MERGED 2026-09-01 (PR #87) behind [ADR-0088](../../decisions/0088-the-mcp-boundary-is-hostile.md): a systematic review of the branch found five merge blockers — an orphaned MCP child on a signal, an optional validated `fetch`, a bypassable transport byte bound, discovery paging past its budget, and two cloud-metadata endpoints reachable through the local opt-in — all reproduced and fixed before it merged. `W5` MERGED 2026-09-02 (PR #88) behind +> [ADR-0089](../../decisions/0089-media-correctness-four-boundaries.md) + +> [ADR-0090](../../decisions/0090-a-continuation-token-rides-the-part-it-belongs-to.md): all six of +> `CR-50`–`CR-55` closed, two of them for one half of a two-part obligation and saying so in their own +> headings. A systematic maintainer review of the branch returned **six merge blockers** — the async +> media-job result bypassing the first-resolution pin, a run-scoped re-host ceiling that reset on resume, a +> capability gate keyed on the model id alone (breaking custom `base_url`), a negative DB media rate buying +> strict-cap headroom, a load-bearing migration backfill with no upgrade test, and a drifted canonical +> AgentRunner spec — all reproduced and fixed before it merged. +> +> **`W6` — authoring correctness — MERGED 2026-09-04 (PR #89)** — `CR-60`–`CR-64` behind +> [ADR-0091](../../decisions/0091-first-means-first-declared-not-first-to-finish.md)–[ADR-0094](../../decisions/0094-a-tool-grant-is-checked-when-the-plan-is-built.md), +> 32 commits — 27 of work over seven internal review rounds, then five folding four maintainer rounds on +> PR #89: `first` means first DECLARED and the plan stops claiming otherwise; a +> tool grant is refused when the plan is built rather than mid-run; an expression sees only its transitive +> closure and a literal out-of-closure read is refused at parse; an `output_schema` is compiled at parse in an +> allowlist-strict mode and enforced at run time — with **no new dependency**, because the compiler the +> deferral said we needed already existed one package away. +> +> **What the wave is actually worth recording for is that most of what the review rounds found were defects in +> the FIXES, not in the code they repaired.** The branch-order search was wrong three times in three rounds, +> each time in the mirror of the previous error. The expression scan's central soundness claim — that it never +> produces a false refusal, which is the entire justification for a regex over a parser — was asserted in three +> documents and false in EIGHT distinct ways — the eighth, a Unicode identifier escape, found after the +> wave was declared complete. One stand-down gate, added to prevent a false refusal, silently +> removed the whole check it was protecting. Every one is a dated correction inside its own ADR rather than a +> quiet rewrite, because the shape of the error is the reusable part. +> +> **The maintainer review then found two merge blockers, and both broke a headline claim of the wave.** Deep +> validation did not enforce `required` when the property's schema constrained nothing — `{ properties: { +> status: {} }, required: ['status'] }` accepted `{}`, because an unconstrained property compiles to +> `z.unknown()`, which is optional inside a `z.object`, and presence was checked only for names ABSENT from +> `properties`. And the expression scanner's regex was quadratic and synchronous: 514 ms at 32 K of +> whitespace, on a parser that accepts 2 MiB, with plan build synchronous so no deadline or cancel could +> interrupt it. Both reproduced and fixed before the merge, with eight further High findings — among them a +> crash on `{ type: 'string', format: 'email', maxLength: 254 }`, which is the exact shape the new canonical +> subset document tells authors to write, caused by an `as unknown as` cast that CLAUDE.md rule 1 forbids +> precisely because it turns a type error into a runtime one. +> +> **Eleven residuals** are recorded in [deferred-tasks.md](../deferred-tasks.md) — ten under the four `W6` +> residual headings plus ADR-0091's zero-branch-merge question, filed with the fan-in items. Two await a +> maintainer ruling. +> +> *(The `W4` start-gate paragraph is kept in [its earlier section](#w4-start-gate--maintainer-ruling-moved-from-currentmd-on-2026-11-04).)* +> +> - **Batch 1, merged 2026-08-11 (PR #82)** — the prerequisite (`#W15-1`), the oracle (`CR-90`, `CR-91`) and +> all of `W0` (`CR-01`–`CR-03`). `CR-64` was added in the same batch and shipped 2026-09-02 with `W6`. +> - **Batch 2, merged 2026-08-24 (PR #83)** — all of `W1`: the eight P0 blockers plus `CR-92`, behind +> ADR-0078…ADR-0084. A comprehensive review of the assembled PR found seven further defects, six of them in +> the W1 code itself; all seven were fixed and mutation-verified before merge. +> +> - **Batch 3, merged 2026-08-28 (PR #85)** — `W2`, liveness and deadlines, behind +> [ADR-0085](../../decisions/0085-the-node-executor-owes-liveness-and-the-engine-enforces-it.md). All six +> items closed; the wave's own closing register is in the phase document. The assembled PR drew a +> **Request changes** verdict with fourteen findings, five of them blockers, plus a later round of eight; +> all were verified against the code before being acted on, and the register records which were fixed and +> which were declined with the reason. The five blockers had one root cause — an ownership decision taken +> at a point in time, guarding work that spans an await. +> Its document review moved the count twice, and the two moves **cancel**: `CR-21` turns out to have +> shipped **with `CR-14`** in PR #83 (ADR-0082 decides both, and the execution graph above already +> scheduled them together) and only its heading was stale, while `CR-21c` — an unbounded individual +> `pollMediaJob` call — was found during the same review and added. `W2` was therefore still **five** open +> items out of six, not four; the phase total moved 47 → 48, not the wave. All five then closed on +> 2026-08-27 — `CR-22`, `CR-21b`, `CR-21c`, and finally `CR-20` and `CR-23` together, which +> [ADR-0085](../../decisions/0085-the-node-executor-owes-liveness-and-the-engine-enforces-it.md) decides as +> one because they are one mechanism at one place. +> +> **Not to be confused with the 2.5.5 remediation's own `W2`** ("Shut the doors", in the graph above). Same +> label, different phase: this one is 2.6.5's liveness wave. +> +> Per-item history, the seven post-review findings and the carried-forward gaps live in the phase document. + +> **Pre-`W7` close-out, 2026-09-06.** *(Superseded in part on 2026-09-14: `CR-96` and `CR-97` reopened exit criterion 1, +> the count is 41 of 51, and `CR-94` is decided and scheduled into `W7` — see the next block.)* The last two non-deferrable items closed, so **exit criterion 1 is met** +> and the phase count is **41 of 48** (40 ✅ headings plus `CR-95`'s short-term half; the arithmetic is shown +> because it has been wrong twice here). `CR-73`: `invoke_agent` was advertised to every model granted it while +> its delegate is wired nowhere in the tree, so the advertise-filter now reads a tool's declared +> `requiresDelegate` and not only its `ToolHost` arm — `read_media` is dropped for the same reason, which is +> `CR-50`'s own argument for leaving it unwired. `CR-80`: a rejected custom `base_url` was caught and the +> DEFAULT adapter left standing, which is the official API; it now installs a refusing adapter that names the +> URL's shape and never its credentials. **Criteria 5 and 7 also closed**: the three missing security sittings +> (prompt/trust provenance, hostile MCP, provider/config trust) and the `W5` and `W6` closing registers are +> written, each stating that it was written after its wave merged rather than with it. **Criterion 6** passes +> on both halves, read from the exit code — `pnpm run ci | tail` reports `tail`'s status and produced one false +> green in this session. Still open: criterion 2 (`CR-94` has no deferral record, and that is a maintainer's +> call) and criterion 4 (the `W3` live blocker keeps the gap-free stream promise wider than the code). +> *(Criteria 2 and 4 have both moved on: `CR-94` was decided 2026-09-13 and its remainder now has a deferral +> record, and ADR-0087 §1 — criterion 4's blocker — is scheduled into `W8`. Criterion 5, called closed above, +> REOPENED on 2026-09-18 with a fifth sitting, `history.db` at rest.)* + +> **`W7` decisions made and accepted, 2026-09-14.** +> +> **What was decided.** The maintainer settled `CR-70`–`CR-72` and `CR-94` on 2026-09-12 and 2026-09-13. Four ADRs +> were **accepted** on 2026-09-14, with their implementation staged in `W7`: +> +> - [ADR-0095](../../decisions/0095-what-an-agent-session-remembers-across-turns.md) decides that tool history is +> persisted as structure, never content; that carrying it into the model's context is deferred; and that `memory` +> is implemented. `memory` stays inert until `W7` lands. +> - [ADR-0096](../../decisions/0096-a-request-is-measured-before-it-is-sent.md) decides that a request is measured +> before it is sent, that a context overflow is classified and recovered only before any tool runs, and that input +> is priced. Until `W7` lands, the estimate still prices output only. +> - [ADR-0097](../../decisions/0097-a-budget-approval-is-an-allowance-not-an-exemption.md) decides that a budget +> approval is a dispatch-owned allowance, shown before it is granted. Until `W7` lands, an approved step still runs +> uncapped. +> - [ADR-0098](../../decisions/0098-a-session-effect-row-holds-no-result-and-never-replays.md) decides that a session's +> effect row holds no result and never replays. +> +> **What the reviews found.** The first six review rounds shaped the ADRs; a seventh, on 2026-09-18, reviewed them +> against the tree before any code (see the block below). The six surfaced two shipping defects, now +> register items `CR-96` and `CR-97`. Both meet the phase's non-deferrable definition, so exit criterion 1 is open +> again, and the count is **41 of 51**. +> +> **What `W7` now covers:** `CR-71`, `CR-72`, `CR-98` (ADR-0096's prerequisites, opened as an item 2026-09-18), +> `CR-94`, `CR-96` and `CR-97`, plus `relavium budget resume`, which moves here from 2.6.K because ADR-0097's +> shown-amount guarantee otherwise has no non-TTY surface. Five canonical claims that were already false are +> narrowed ahead of the code: +> +> - the session spec's tool history and export `tools` union; +> - `memory`'s default; +> - the uncapped approved budget step; +> - the SSE schema's "user/assistant/tool messages are persisted" sentence (2026-09-18); +> - the chat reference's "full transcript" in the export (2026-09-18). +> +> *(This `W7` is Phase 2.6.5's seventh wave. It is not the remediation's "Wave 7 — Orchestration & the gate" in +> the graph at the top of this file, exactly as `W2` is not Wave 2.)* + +> **`W7` document review, 2026-09-18 — before any code.** Eight dimensions reviewed the four ADRs, the register, +> the deferrals and every canonical document they touch against the tree; each finding was verified by an +> independent adversarial pass. 113 findings held, and the maintainer settled seventeen decisions the same day. +> Every ADR carries a dated note; the register carries a per-document landing checklist. Three document defects +> mattered most, and all three would have been discovered mid-implementation: +> +> - **ADR-0098's join does not exist.** It says the effect journal already records the tool-call id; every session +> surface writes a wiring constant instead, so `W7` must thread an engine-assigned id through the dispatch port. +> - **ADR-0098's monotonic turn key could not be derived** from the journal, because the sweep deletes the rows the +> high-water mark would be read from. It becomes a durable column on the session row. +> - **ADR-0096 §1's per-part floor** would have charged an inline image megabytes of tokens, contradicting the +> per-modality media ceiling in the same section. +> +> Two of the seventeen decisions change what ships beyond those ADRs: `history.db` opens with +> `PRAGMA secure_delete = ON`, so "no tool result at rest" is true of the file's bytes and not only of the row; and +> a pre-content HTTP 4xx now releases its admission instead of committing an input-sized estimate, which keeps a +> rate-limit burst from exhausting a session's cap once input is priced. ADR-0087 §1 — the `W3` live blocker that +> belonged to no wave — is scheduled into `W8`. + +**This is the corrected execution order, and it is what the graph above shows:** + +1. **`#W15-1` first.** Its five staged steps touch `run-event.ts`, `engine.ts`, `checkpoint.ts` and + `run-history-store.ts` — the same four files `CR-10` and `CR-12` restructure. Landing it after the durability + spine means writing its barrier against a persistence path that is about to change. +2. **Then Phase 2.6.5, closing before Wave 2 opens.** Which is only true if the work Wave 2 would block on + lives there — so **the hostile-MCP threat class moves into 2.6.5**: `CR-16` (consent before a stdio spawn) + plus `CR-40`–`CR-42` (transport cancellation, DNS/redirect SSRF, ingress bounds). Wave 2's queue item 2 is + **executed as 2.6.5's `W4`**; the 2.5.5 finding ids keep their home in the 2.5.5 phase doc and certify from + there. Wave 2 keeps the MCP name-collision/tool-poisoning trust items, the fs jail, the secrets layer, + persistence-security and the certification. +3. **Within 2.6.5, the oracle comes before the spine.** `CR-90`/`CR-91` (crash + durable-truth harness) land + first because they are the instrument `CR-10 → CR-11 → CR-92 → CR-12` is proven with; `CR-92` moved out of + the harness group into the spine, since an API returning `completed` while history says `failed` is a + runtime defect, not a test concern. + +Two rules that changed with the plan review: + +- **Nineteen items are non-deferrable** — every W0/W1 item plus `CR-50`, `CR-55`, `CR-73`, `CR-80`, `CR-92`, `CR-95`'s short-term fix, and (added 2026-09-14) `CR-96` and `CR-97`. Each has a cheap fail-closed option (refuse, remove, narrow the claim), so "too big + to fix now" argues for the cheap option, never for deferral. The previous exit criterion permitted deferring + all 43 items and declaring the phase complete. +- **The gate is `pnpm run ci` AND `pnpm coverage`.** `coverage` is a separate required CI check and is not + inside the `ci` script; this phase edits `packages/core` heavily, and the CI coverage job does not block on + `core`. + +`CR-01`–`CR-03` were half-closed by Wave 1 and finished right after the oracle — **all three are closed as of +2026-08-11** (PR #82). `CR-03` was a propagation gap from `#W15-10`'s own fix: the finding named three `--json` +paths that never got the safe serializer, and closing it found **five**, which is why the call-site half is now +an ESLint selector rather than a list. + + +### [Systematic group 2a, round 3 — from current.md's “Not started yet”] + +**Systematic group 2a, round 3 — 2026-10-04.** [Two complete independent reviews](https://github.com/HodeTech/Relavium/blob/f267bb0d4e9a4b4e2b51cb2c44ff744284d5228a/docs/reviews/2026-10-04T15-44-35-w7-systematic-group-2a-round-3-review.md) +verify all four tool notifications, then reproduce an inherited raw-callback usage/hard-cap gap and +skipped realized recording. `e7973031` preserves original error identity while carrying canonical +accounting and starts mandatory recording after authoritative cost aggregation despite delivery failure. +Original baseline: 40 failing/13 passing; corrected root: 7,625 passing/12 skipped across 331 files, +23 forced tasks and six forced builds. Every sealed report/entry is verified before Original resumes. +**Fresh complete round 4 is required; group 2a, ownership, remaining groups, Steps 7–8/12, Sonar and PR remain open.** diff --git a/docs/runbooks/local-dev-setup.md b/docs/runbooks/local-dev-setup.md index 0c7c6fe4..ed51bbe2 100644 --- a/docs/runbooks/local-dev-setup.md +++ b/docs/runbooks/local-dev-setup.md @@ -1,6 +1,6 @@ # Local Dev Setup -> Last updated: 2026-06-03 +> Last updated: 2026-11-04 This runbook gets the Relavium monorepo building on your machine. Relavium is a **Turborepo + pnpm** TypeScript monorepo with a thin Rust layer for the Tauri desktop @@ -63,28 +63,21 @@ pnpm --filter @relavium/core test Once the engine is green, bring up a surface. Build order is CLI → desktop → extension (see [architectural-principles.md](../standards/architectural-principles.md) §1). -**CLI (fastest feedback loop):** +**CLI (fastest feedback loop):** the CLI has no watch-mode script; build it, then run the bundle. ```bash -pnpm --filter relavium dev +pnpm turbo run build --filter=relavium # then, against a committed workflow file: -relavium run ./workflows/.relavium.yaml --input topic= +node apps/cli/dist/index.js run ./workflows/.relavium.yaml --input topic= ``` -**Desktop app (Tauri v2, needs the Rust toolchain):** +Rebuild after every source change: the bundle is what runs, so a stale `dist/` runs old code. To call it +as `relavium`, put an alias or a link to `apps/cli/dist/index.js` on your `PATH`. -```bash -pnpm --filter @relavium/desktop tauri dev -``` - -This starts the Vite + React 19 frontend and the Rust backend together with hot reload. - -**VS Code extension:** - -```bash -pnpm --filter @relavium/vscode-extension build -# then press F5 in VS Code on apps/vscode-extension to launch an Extension Host -``` +**Desktop app and VS Code extension:** not in the repository yet. `apps/desktop` and +`apps/vscode-extension` hold only a README; the desktop app is built in +[Phase 3](../roadmap/phases/phase-3-desktop.md) and the extension in +[Phase 4](../roadmap/phases/phase-4-vscode.md), and this section gains their commands then. ## 4. Add a provider key for local runs @@ -100,8 +93,8 @@ in the repo. | 2 | `packages/llm` | `shared` | | 3 | `packages/core` | `shared`, `llm` | | 4 | `apps/cli` | `core` | -| 5 | `apps/desktop` + `packages/ui` | `core` | -| 6 | `apps/vscode-extension` | `core`, `ui` | +| 5 | `apps/desktop` + `packages/ui` | `core` — **Phase 3; not in the repository yet** | +| 6 | `apps/vscode-extension` | `core`, `ui` — **Phase 4; not in the repository yet** | | 7 | `apps/api` + `apps/portal` | `core`, `ui` — **Phase 2 (cloud) only** | ## Troubleshooting diff --git a/docs/standards/code-review.md b/docs/standards/code-review.md index 7c27e9c3..ee1a9650 100644 --- a/docs/standards/code-review.md +++ b/docs/standards/code-review.md @@ -11,8 +11,16 @@ are actually enforced — automation catches format and types; a human catches t ## Ground rules -- Nothing merges to the default branch without a review approval and green required CI - (typecheck, lint, unit tests, fixture-mode conformance — see [testing.md](testing.md)). +- Nothing merges to the default branch without a recorded review and green required CI + (typecheck, lint, unit tests, fixture-mode conformance — see [testing.md](testing.md)). While the + project has one maintainer, GitHub cannot enforce an approval (it refuses a self-approval), so branch + protection requires the CI checks and the review is the maintainer's own plus an independent review + round recorded for the change under the roadmap's acceptance bar. A GitHub approval becomes a required + check when a second maintainer joins. + + > Amended 2026-11-04: "a review approval" read as a GitHub approval that `main`'s protection has never + > required (0 approvals, no CODEOWNERS); a PR #96 review found the mismatch and the maintainer ruled + > the rule should state the practice. - The author runs lint, format, and tests **before** requesting review; review is not a substitute for the toolchain. - Keep PRs small and scoped to one concern; a reviewable diff gets a real review. Large diff --git a/docs/standards/error-handling.md b/docs/standards/error-handling.md index 2331e861..9c18b83f 100644 --- a/docs/standards/error-handling.md +++ b/docs/standards/error-handling.md @@ -128,111 +128,8 @@ It is a defense-in-depth **backstop**: the adapter still owns passing an already message, and the backstop masks a finite set of secret **shapes**, not arbitrary sensitive content. -- **Shapes.** OpenAI/Anthropic `sk-…` and Google `AIza…` key prefixes; a finite list of literal - secret query names, after `?`, `&` or `#` (an OAuth implicit-flow `#access_token=`); the exact - `api-key`, `x-api-key` and `x-goog-api-key` header names, also in quoted, JSON-encoded and - JSON-in-JSON (`\"x-api-key\":\"…\"`) objects; and `Bearer`/`Basic` authentication values. A header - value may begin with up to eight (possibly quoted, escaped, punctuation-glued or LF/TAB-split) - `Bearer`/`Basic` labels — a doubled `Bearer Bearer` is a common misconfiguration. Its token may - begin with any run of (escaped) quotes and ends at a delimiter, at escapes before one, or before a - nested header name, which then masks its own value. A label may be closed by its own quote before - its token (`"Bearer" `), and a run of up to eight concatenated labels (`BearerBearer`) - counts as one label. No label takes as its token another label that whitespace or the end of the - text follows, even one glued behind punctuation, quoted or concatenated, so a standalone value - with any number of separately stacked labels masks the token after the last one, and a label glued - to a token behind a non-word character masks its own token; a second label that other punctuation - follows (`Bearer Bearer;`) is masked as a token, which only over-masks. -- **Masking.** Every rule, the literal query names included, finds its matches in the original - text, and the masked ranges of all rules are merged before anything is replaced, so no rule - can erase the context another rule needs. A header, label or query rule masks only its token: - the header name, the separator, any labels and the query name stay visible, an LF/TAB-split - label keeps its spelling, and quotes around a token stay outside it. Masking a token that is - already exactly `[REDACTED]` changes nothing, so a masked token is a fixed point, quoted, - escaped or after labels. After a `Bearer`/`Basic` label, text glued to the marker - (`Bearer [REDACTED];tail`) is masked with it; a header token ends at `,`, `;` or `}`. One - pass can still glue a marker to the text after another rule's token, so the scrub repeats - until its output stops changing, at most three passes, and blanks the whole diagnostic if it - has not settled: a second scrub changes nothing. -- **URL userinfo.** The `?`/`#` rule decides where userinfo ends: a raw `?` or `#` before the - authority `@` begins public query or fragment text only when the text before it is host-shaped (a - dotted name or IPv4, `localhost` or a bracketed IP literal, with an optional port, at most 260 - characters in all) **and** an `=` assignment follows it, as in - `https://host.example?contact=a@b.example`. Masked userinfo that holds such a raw `?` or `#` - leaves the authority ambiguous: unless a host-shaped name follows its last `@`, the text there may - be the rest of a query value (`https://svc?password=P@ss`), so the mask runs through the - authority's end, the next `/`, whitespace, quote, `,` or `}`. The scheme rule then decides what is - masked: `http(s)`, `ws(s)` and `ftp(s)` mask any userinfo, while the connection-string schemes - `postgres`, `postgresql`, `mysql`, `mariadb`, `redis`, `rediss`, `mongodb`, `mongodb+srv`, `amqp`, - `amqps`, `ssh`, `sftp`, `ldap`, `ldaps`, `smtp` and `smtps` mask only colon-bearing userinfo - (`user:pass@`, `:pass@`), so a bare user such as `ssh://git@host` (or a colonless - `redis://#TOKEN@host`) stays visible. Userinfo whose password is exactly masked stays as it is: - beside any username under a connection-string scheme, or beside a masked or empty username under - any scheme; an empty password, or text glued after a masked one, is still masked. The - authored-content detector's projection of `postgres://app:{{secrets.password}}@db` to - `postgres://app:[REDACTED]@db` is therefore a fixed point. Core's journal projection applies the - same `?`/`#` predicate and also merges its rules' ranges, but masks a userinfo together with the - values its key and opaque-query rules recognize in it, its username's excepted, each to its own - end, and ends a value no rule recognizes at the last `@`, whose text after it stays as the host, - rather than masking through the authority's end - ([built-in tools](../reference/shared-core/built-in-tools.md#finite-diagnostic-credential-masking)). -- **Split and normalized forms.** Every rule except the literal query names also probes the - terminal-normalized text; a credential revealed only by normalization masks the **whole** - diagnostic. The key prefixes, URL userinfo and the exact header names are additionally probed in - the LF/TAB-joined view at original-text boundaries — of the raw text too when terminal - normalization changed it, since a terminal sequence there may take a split credential's first - letter (ESC `A` before `AIza…`) — and a credential revealed only by such a split masks the whole - diagnostic too. A `Bearer`/`Basic` label may itself be split by LF/TAB. -- **Documented trade-offs, all failing closed.** A scheme glued to a word character is masked in - place; an unlisted scheme ending in a listed one (`gftp://`, `jdbc:mysql://`) matches by its - suffix; prose that the whitespace-stripped residual probe glues to a path-less URL and a later `@` - blanks the whole diagnostic, and so does a key prefix (`sk-`, `AIza`) that the same probe joins - with the following words to its minimum length, at the start of the text or, since the scrub - repeats, right after another rule's mask (`Authorization: Bearer xyz sk-ant api key invalid`); a - path-less terminal hyperlink target (OSC 8) that the raw LF/TAB-joined probe joins with a later - `@` (a hyperlinked table cell before an e-mail address) blanks it as well; a host fragment with no - assignment (`https://host.example#sec@x`), a single-label host query and an over-long host-shaped - prefix are masked, and so is the text after the last `@` of such an ambiguous authority unless it - is host-shaped; a `Bearer`/`Basic` label that ends a line takes the next token with it; an opaque - value after a label ends only at whitespace, so in compact JSON it masks to the end of the run; - nine or more concatenated labels, read in the raw text as well as the normalized one, and a token - that begins with `,`, `;` or `}` after a `Bearer`/`Basic` label in a header value, blank the whole - diagnostic. Past eight separate labels after a header name, and in a value of labels alone glued - behind punctuation (`Bearer |Bearer`), the outcome depends on what precedes the header name: when - its nearest non-whitespace character is a word character, `-` or `_`, the diagnostic stays in - place (the last label masks any token); otherwise, the start of the text included, it is blanked. - No token is visible either way, and the output is stable under a second scrub. -- **Limits.** Userinfo under an unlisted scheme is not masked, including driver-qualified spellings - such as `postgresql+psycopg2://`. Opaque `Bearer`/`Basic` values, `api-key` header values and - literal secret query values end at any whitespace, LF and TAB included, so a fragment after such a - break stays visible, as it does after an ordinary space; a label glued to a quoted token without - whitespace (`Bearer""`) is likewise outside the shape, as is a token after a JSON escape - (`\n`, `\t`) that stands for the whitespace after a label. A JSON escape right after a header - name's colon starts its token, so a value glued to the escape is masked with the escape's letter - (`"x-api-key":"\n"` → `"x-api-key":"\[REDACTED]"`), while a quoted value behind such an escape - stays visible (`x-api-key:\t""`). A terminal sequence with a parameter whose final byte is a - shape's first letter (ESC `[` `1` `B` before `earer `), and a terminated device-control, - start-of-string or application-program string (ESC `P`, ESC `X` or ESC `_`, ended by ST or BEL) - glued before a shape, hide that shape from every view: in the raw text the sequence's last byte - glues to the shape's first letter, and normalization removes it. A word glued before `sftp://` on - the same line (`statusftp://user@h`) reads as `sftp`, so its colonless userinfo stays visible like - any `sftp` user: every whitespace-joined probe, the CLI's display residual among them, glues prose - to a scheme the same way, and reading it as `ftp` would blank an ordinary - `connect to sftp://user@host` diagnostic. (A word that ends a line before `ftp://` is no such - glue: an LF/TAB-split ftp credential after it blanks the diagnostic.) Malformed URLs - (`https:/u:p@h`), a raw `/` inside a password, invisible characters inside a scheme and JSON whose - quotes are themselves escaped as `\u0022` are outside the finite shapes. In a JSON-encoded - diagnostic, an escape written as two characters (`\n`, `\t`, `\r`) and glued before a key prefix, - a header name or a label defeats that shape's word boundary, so the credential after it stays - visible (`"key rejected:\nsk-…"`). Core's journal projection shares this limit for key prefixes - and scheme labels, while core's summary also reads each such escape as its separator and masks the - credential after it; core's key/value rule's word-character prefix still reads a header or - secret-key name glued after such an escape (`"err:\napi-key: …"`) and masks the value with it. A header value - that begins with `,`, `;` or `}` and has no label before it is an empty token, so the text after - it stays visible. The backstop has no generic key/value rule: a secret key inside a URL authority - whose value holds `@` — after `;` (`https://svc;a=1;password=P@ss…`), after `&` unless it is one - of the literal secret query names, or on the next line of a JSON-encoded file whose break is the - escape `\n` — leaves its value's text after the last `@` visible as the host. Core's key/value - rule reads that key and masks its value. +The finite shape list, with its documented limits, is specified in +[diagnostic-secret-backstop.md](../reference/shared-core/diagnostic-secret-backstop.md). ## No silent catches diff --git a/packages/llm/src/scrub-secrets.ts b/packages/llm/src/scrub-secrets.ts index 716d14a1..db8bf7b4 100644 --- a/packages/llm/src/scrub-secrets.ts +++ b/packages/llm/src/scrub-secrets.ts @@ -12,8 +12,9 @@ const MASKED = '[REDACTED]'; * the engine's `deInlineMedia` pass, leak-freedom here is an active transform at one boundary, not * per-adapter discipline. It masks finite common secret **shapes**: prefixed keys, literal secret query * names, URL userinfo of a finite scheme list and finite API-key/Bearer/Basic headers. Each non-query rule probes - * the original diagnostic independently, including terminal-normalized and LF/TAB-joined views, before any - * other rule can erase its identifying context. A match revealed by normalization masks the whole + * the original diagnostic independently, including the terminal-normalized view, and the key prefixes, URL + * userinfo and exact header names the LF/TAB-joined view too, before any other rule can erase its identifying + * context; an opaque Bearer/Basic value split by an LF or TAB keeps its tail (a documented limit). A match revealed by normalization masks the whole * diagnostic. The finite patterns check boundaries in the original text (the URL rule has none, like the * pre-PR matcher, so a scheme glued to a word character is masked in place) and preserve already-masked spans; * this is not universal fragment discovery. Query names are excluded from the residual probe, but their @@ -24,7 +25,8 @@ const MASKED = '[REDACTED]'; * check, with no parsing or external calls on the error path. * Asserted by `llm-error.test.ts` (the backstop) and each adapter's * secret-safety test (a planted secret → a secret-free surfaced `LlmError`); see security-review.md. - * The canonical policy is error-handling.md's diagnostic secret backstop. + * The rule is error-handling.md's diagnostic secret backstop; the finite shape list and its limits are + * specified in docs/reference/shared-core/diagnostic-secret-backstop.md. */ export function scrubSecrets(text: string): string { return toFixedPoint(scrubOnce, text, MASKED); @@ -129,7 +131,7 @@ const NON_QUERY_SECRET_RULES: readonly SecretRule[] = [ // at a delimiter, at escapes before one or before a header name glued to it, which then masks its // own value. A JSON escape after the `:` starts the token, so it takes a value glued to the // escape (`"x-api-key":"\n"` keeps only the backslash) but not a quoted value behind one - // (`x-api-key:\t""`), a documented limit (error-handling.md). Every label repetition is + // (`x-api-key:\t""`), a documented limit (diagnostic-secret-backstop.md). Every label repetition is // bounded so no label run can exhaust the RegExp stack: past eight separate labels the standalone // label rule masks the token after the last one, and nine concatenated labels blank the diagnostic // (`LABEL_RUN_PAST_BOUND`). From eb2cce487e4a38a1a93af91f2a9a02b6783c5d04 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Wed, 4 Nov 2026 15:16:02 +0300 Subject: [PATCH 95/97] docs(repo): add the publish-credential prerequisite to MVP-A23 The PR #96 ruling on NPM_TOKEN said the release prerequisites would carry it: the publish credential is scoped to the release job before the guard lifts, by npm trusted publishing or a protected `release` environment that holds the token, with the repository-level secret deleted. The D59 release ADR chooses between them. Co-Authored-By: Claude Opus 5.5 --- docs/roadmap/phases/phase-2.6-conversational-authoring.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/roadmap/phases/phase-2.6-conversational-authoring.md b/docs/roadmap/phases/phase-2.6-conversational-authoring.md index c7f810ad..67882d26 100644 --- a/docs/roadmap/phases/phase-2.6-conversational-authoring.md +++ b/docs/roadmap/phases/phase-2.6-conversational-authoring.md @@ -1362,7 +1362,7 @@ workstream named in its row and follows that workstream's ADR and security gates | MVP-A20 | 2.6.I | **Masked `provider set-key` on a TTY** instead of refusing an interactive terminal. Secret-input sitting. | | MVP-A21 | 2.6.I | **Registered MCP servers reach the default agent**, under ADR-0084 spawn consent and ADR-0088's untrusted tool definitions. | | MVP-A22 | 2.6.F | **Upgrade from `0.1.1`.** A populated `0.1.1` home (history.db with migrations 0000–0004, config.toml, keychain entries) upgrades through v0.2.0, with a seeded-then-migrated test. | -| MVP-A23 | 2.6.F | **Release prerequisites for v0.2.0.** The D59 release ADR and guard change; version and CHANGELOG including the Node ≥22 break; npm token, 2FA, provenance and the `v*` ruleset verified; the interactive Home and chat checked on a real Windows terminal; a manual three-provider end-to-end turn; first-read docs (README, npm README, tutorials, runbooks) describing what ships. | +| MVP-A23 | 2.6.F | **Release prerequisites for v0.2.0.** The D59 release ADR and guard change; version and CHANGELOG including the Node ≥22 break; npm token, 2FA, provenance and the `v*` ruleset verified; the publish credential scoped to the release job, either npm trusted publishing (OIDC, no long-lived token) or the `release` deployment environment with a `v*`-tag deployment policy and a required reviewer holding `NPM_TOKEN`, with the repository-level secret deleted (PR #96 review, 2026-11-04; chosen in the D59 release ADR); the interactive Home and chat checked on a real Windows terminal; a manual three-provider end-to-end turn; first-read docs (README, npm README, tutorials, runbooks) describing what ships. | ## Deferred-tasks pulled into this phase From 062564f0ab0b472b6a579f104a050e8f9d77d0b4 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Wed, 4 Nov 2026 16:19:46 +0300 Subject: [PATCH 96/97] refactor(core): keep the review fixes inside the complexity bound SonarCloud's re-analysis of PR #96 (gate OK) reported two cognitive complexity increases from this correction: redactArgVector rose past the bound (21) and visitUrlUserinfo from 18 to 21. The inline-credential and comma/slash-password branches move into helpers with identical behavior, and two cases now pin what the move must keep: a username key does not swallow the host, and with no `@` to read on to the scan resumes inside the authority. The keyring stub gains the real Entry's refusing methods instead of a NOSONAR comment Prettier had moved off its line (S2094). Every mutant of the helpers is killed. Co-Authored-By: Claude Opus 5.5 --- packages/core/src/tools/bounding.test.ts | 4 ++ packages/core/src/tools/bounding.ts | 62 +++++++++++++-------- tools/cli-package/frame-capture-keyring.mjs | 13 ++++- 3 files changed, 54 insertions(+), 25 deletions(-) diff --git a/packages/core/src/tools/bounding.test.ts b/packages/core/src/tools/bounding.test.ts index bb2d59d2..edcfd70c 100644 --- a/packages/core/src/tools/bounding.test.ts +++ b/packages/core/src/tools/bounding.test.ts @@ -1360,6 +1360,10 @@ describe('PR95 raw ?/# userinfo in the journal projection', () => { ['https://user:pa/ss@host.example/@scope/pkg', 'https://[redacted]@host.example/@scope/pkg'], // A raw `?` that is not a host's public query stays userinfo, as the authority scan reads it. ['https://u:p?x/y@h.example/', 'https://[redacted]@h.example/'], + // A key in the username names the user, so its value stops at the `@` and the host stays. + ['https://x-access-token:pa,ss@github.com/o/r', 'https://[redacted]@github.com/o/r'], + // With no `@` to read on to, the scan resumes inside the authority and finds the later scheme. + ['https://xhttps://user:pw@h.example/', 'https://xhttps://[redacted]@h.example/'], ] as const) { expect(redactSecretShapedText(input), input).toBe(expected); } diff --git a/packages/core/src/tools/bounding.ts b/packages/core/src/tools/bounding.ts index 9d86580a..ba81425c 100644 --- a/packages/core/src/tools/bounding.ts +++ b/packages/core/src/tools/bounding.ts @@ -1176,6 +1176,24 @@ function passwordPastDelimiter(text: string, start: number, end: number): number return end + tail.length - 1; } +/** + * Mask a userinfo whose password holds a `,` or `/` ({@link passwordPastDelimiter}); its username owns the + * text before the first `:`, as in {@link visitUrlUserinfo}. Returns where the scheme scan resumes: after + * the `@` it masked through, or at `start` when there is none. + */ +function maskPasswordPastDelimiter( + text: string, + schemeStart: number, + start: number, + end: number, + mask: SecretSpanVisitor, +): number { + const passwordAt = passwordPastDelimiter(text, start, end); + if (passwordAt < 0) return start; + mask(start, passwordAt, '[redacted]', { schemeStart, ownsUntil: text.indexOf(':', start) }); + return passwordAt + 1; +} + /** A password's tail past a `,` or `/`, through its `@`; bounded so a long run is never rescanned. */ const PASSWORD_TAIL = /[^\s"'}\]?#@]{0,512}@/y; @@ -1192,13 +1210,7 @@ function visitUrlUserinfo(text: string, mask: SecretSpanVisitor): void { const authorityAt = firstQuery < 0 ? -1 : authority.lastIndexOf('@', firstQuery - 1); const at = authorityAt >= 0 ? authorityAt : authority.lastIndexOf('@'); if (at < 0) { - const passwordAt = passwordPastDelimiter(text, start, end); - if (passwordAt >= 0) { - // Its username owns the text before the first ':', as below. - const url = { schemeStart: match.index, ownsUntil: start + authority.indexOf(':') }; - mask(start, passwordAt, '[redacted]', url); - scheme.lastIndex = passwordAt + 1; - } + scheme.lastIndex = maskPasswordPastDelimiter(text, match.index, start, end, mask); continue; } const userinfo = authority.slice(0, at); @@ -1447,28 +1459,32 @@ function redactArgVector( redactNext = undefined; continue; } - if (typeof item === 'string') { - const separator = item.indexOf('='); - const flag = separator > 0 ? item.slice(0, separator) : ''; - if (isCredentialFlag(flag)) { - const masked = redactCredentialArgument(flag, item.slice(separator + 1), whole); - if (masked !== undefined) { - out.push(`${flag}=${masked}`); - continue; - } - } - const glued = gluedUserPassword(item); - if (glued !== undefined) { - out.push(`-u${whole(glued, '[redacted]')}`); - continue; - } - if (isCredentialFlag(item)) redactNext = item; + const inline = typeof item === 'string' ? redactInlineCredential(item, whole) : undefined; + if (inline !== undefined) { + out.push(inline); + continue; } + if (typeof item === 'string' && isCredentialFlag(item)) redactNext = item; out.push(redactSecretShapedWalk(item, seen, project, collide, whole)); } return out; } +/** + * One argument that carries its own credential, masked: a credential flag's `--flag=value` spelling or curl's + * glued `-ualice:secret`. `undefined` when the argument is neither, so the vector walks it as usual. + */ +function redactInlineCredential(item: string, whole: WholeMask): string | undefined { + const separator = item.indexOf('='); + const flag = separator > 0 ? item.slice(0, separator) : ''; + if (isCredentialFlag(flag)) { + const masked = redactCredentialArgument(flag, item.slice(separator + 1), whole); + if (masked !== undefined) return `${flag}=${masked}`; + } + const glued = gluedUserPassword(item); + return glued === undefined ? undefined : `-u${whole(glued, '[redacted]')}`; +} + /** * One credential-value rule for both argv spellings; only ambiguous -p may name a public port. `undefined` * means the value is not a credential after all (a `-u`/`--user` name without a password) and is walked as diff --git a/tools/cli-package/frame-capture-keyring.mjs b/tools/cli-package/frame-capture-keyring.mjs index d4b74c07..e183798d 100644 --- a/tools/cli-package/frame-capture-keyring.mjs +++ b/tools/cli-package/frame-capture-keyring.mjs @@ -3,12 +3,21 @@ const refused = () => { throw new Error('The symbolication run refuses the keyring'); }; -// A class, not a function, so the exports below extend it as the real package's classes do. +// A class, not a function, so the exports below extend it as the real package's classes do; its methods +// mirror the real `Entry`'s and refuse too, although the constructor already has. export class Entry { - // NOSONAR — see above constructor() { refused(); } + getPassword() { + return refused(); + } + setPassword() { + return refused(); + } + deletePassword() { + return refused(); + } } export class AsyncEntry extends Entry {} export class EntryTask extends Entry {} From f25fb83da17583c6e51ee72c7d5bfdbdfc7ad5a7 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Wed, 4 Nov 2026 17:30:51 +0300 Subject: [PATCH 97/97] refactor(core): pass the media byte counter unconditionally The PR #96 review fix that counts omitted inline media against the model ceiling made the counter optional, and its three guards lifted redactInlineMediaForText's cognitive complexity to 18 (SonarCloud S3776, new on 2026-11-04). The observability projection now passes a discarded counter instead, so the walk keeps its pre-review shape. No behavior change: toText always passed a counter, and the event-field projection ignores it. Co-Authored-By: Claude Opus 5.5 --- packages/core/src/tools/bounding.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/core/src/tools/bounding.ts b/packages/core/src/tools/bounding.ts index ba81425c..5ce62815 100644 --- a/packages/core/src/tools/bounding.ts +++ b/packages/core/src/tools/bounding.ts @@ -76,7 +76,7 @@ function isPlainObject(value: object): value is Record { * sees only a flat string). Display-only — the dispatch already ran on the real args. Cycle-safe. */ export function redactInlineMedia(value: unknown): unknown { - return redactInlineMediaForText(value, new WeakSet()); + return redactInlineMediaForText(value, new WeakSet(), { bytes: 0 }); } /** @@ -92,11 +92,11 @@ interface OmittedMedia { function redactInlineMediaForText( value: unknown, seen: WeakSet, - omitted?: OmittedMedia, + omitted: OmittedMedia, ): unknown { if (typeof value === 'string') { if (!isBase64DataUri(value)) return value; - if (omitted !== undefined) omitted.bytes += utf8ByteLength(value); + omitted.bytes += utf8ByteLength(value); return '[base64 data URI omitted]'; } if (typeof value !== 'object' || value === null) { @@ -111,7 +111,7 @@ function redactInlineMediaForText( return value.map((item) => redactInlineMediaForText(item, seen, omitted)); } if (isBinaryBuffer(value)) { - if (omitted !== undefined) omitted.bytes += binaryByteLength(value); + omitted.bytes += binaryByteLength(value); return '[binary buffer omitted]'; // a raw Uint8Array/ArrayBuffer — never JSON the decimal byte values } if (!isPlainObject(value)) { @@ -120,7 +120,7 @@ function redactInlineMediaForText( if (isCanonicalBase64Source(value)) { const data = value['data']; const base64Length = typeof data === 'string' ? data.length : 0; - if (omitted !== undefined) omitted.bytes += base64Length; + omitted.bytes += base64Length; // Report the base64 CHARACTER length (≈1.33× the byte count), not "bytes" — observability only. return { kind: 'base64', base64Length }; }