diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f129dc..5ead142 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,10 +27,16 @@ jobs: - run: scriptorium --version - run: scriptorium init --help - run: scriptorium inventory --help + - run: scriptorium migrate --help + - run: scriptorium resume --help - run: scriptorium status --help - run: scriptorium doctor --help - run: scriptorium host install --help + - run: scriptorium components --profile core --json + - run: scriptorium install core --target ./scriptorium-components-preview --json - run: python -m unittest discover -s tests -v + - name: Run general project intake acceptance + run: python tests/e2e_general_intake.py e2e-windows: runs-on: windows-latest @@ -39,27 +45,76 @@ jobs: with: path: scriptorium persist-credentials: false + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: v0.1.0 + path: scriptorium-v0.1.0 + persist-credentials: false - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: repository: scriptorium-suite/scriptorium-spec - ref: 66bfefea93758231491fadc4694708d1b8107ea0 + ref: 36612e80c49f5a8f346a09ed73f4b1c67cef5135 path: scriptorium-spec persist-credentials: false - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: repository: scriptorium-suite/steward - ref: 21e4e454fb97702bc5f31c24b805cf3c9489bc31 + ref: 7aafad1b432207905072c769fe527a3d4d1b365e path: steward persist-credentials: false - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: repository: foxsplendid/Provenance - ref: 910efbc47e604c51314866174581cfdd7eac47b6 + ref: 1e3f1c022148c33f257d8a27502b50dbe906611d path: Provenance persist-credentials: false + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: foxsplendid/Academic-Slides-Agent + ref: 6371f1e6eac41f505aed39e58dafb689317cab65 + path: Academic-Slides-Agent + persist-credentials: false - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" + - name: Install pinned build backend + run: >- + python -m pip install + --disable-pip-version-check + --no-deps + --only-binary=:all: + "setuptools==83.0.0" + - name: Verify clean Windows install lifecycle + shell: pwsh + run: | + python .\scriptorium\tests\e2e_install_lifecycle.py ` + --scriptorium-root .\scriptorium ` + --spec-root .\scriptorium-spec ` + --steward-root .\steward ` + --provenance-root .\Provenance ` + --previous-scriptorium-root .\scriptorium-v0.1.0 ` + --require-windows ` + --report .\install-lifecycle-report.json + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + $report = Get-Content -LiteralPath .\install-lifecycle-report.json -Raw | ConvertFrom-Json + if ( + $report.status -ne "passed" -or + $report.lifecycle.clean_install -ne "passed" -or + $report.lifecycle.uninstall -ne "passed" -or + $report.lifecycle.reinstall -ne "passed" -or + $report.lifecycle.standalone_components -ne "passed" -or + $report.lifecycle.doctor_and_demo -ne "passed" -or + $report.lifecycle.version_transition.status -ne "passed" -or + $report.lifecycle.version_transition.previous_version -ne "0.1.0" + ) { + throw "Clean Windows install lifecycle did not pass" + } + - name: Install uv for Lectern + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + with: + version: "0.11.16" + python-version: "3.12.3" - name: Install source checkouts shell: pwsh run: | @@ -67,6 +122,8 @@ jobs: .\.demo-venv\Scripts\python.exe -m pip install --no-deps .\scriptorium .\.demo-venv\Scripts\python.exe -m pip install --no-deps .\steward .\.demo-venv\Scripts\python.exe -m pip install --no-deps .\Provenance + - name: Install Lectern workspace without provider extras + run: uv sync --project .\Academic-Slides-Agent --all-packages --locked --no-dev - name: Smoke installed public pull entries shell: pwsh run: | @@ -78,6 +135,13 @@ jobs: if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } & .\.demo-venv\Scripts\prov-sync-unresolved.exe --help | Out-Null if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + & .\.demo-venv\Scripts\prov-ingest-research.exe --help | Out-Null + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $contextVersion = & .\.demo-venv\Scripts\prov-context.exe --version + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + if (($contextVersion -join "`n").Trim() -ne '0.18.0') { + throw "Installed Provenance context runtime version mismatch" + } $raw = & .\.demo-venv\Scripts\prov-sync-pull.exe --capabilities --json $code = $LASTEXITCODE @@ -98,8 +162,12 @@ jobs: if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } & .\.demo-venv\Scripts\scriptorium.exe inventory --help | Out-Null if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + & .\.demo-venv\Scripts\scriptorium.exe migrate --help | Out-Null + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } & .\.demo-venv\Scripts\scriptorium.exe status --help | Out-Null if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + & .\.demo-venv\Scripts\scriptorium.exe resume --help | Out-Null + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } $errorRaw = & .\.demo-venv\Scripts\scriptorium.exe pull --json 2>$null $errorCode = $LASTEXITCODE @@ -163,12 +231,148 @@ jobs: if ([IO.File]::ReadAllText((Join-Path $source 'private-note.md')) -ne 'PRIVATE_RESEARCH_SENTINEL') { throw "Inventory modified a source file" } + - name: Smoke installed synthetic migration lifecycle + shell: pwsh + run: | + $root = Join-Path $env:RUNNER_TEMP 'migration smoke' + $source = Join-Path $root 'selected source' + $workspace = Join-Path $root 'research workspace' + $env:LOCALAPPDATA = Join-Path $root 'canonical local state' + New-Item -ItemType Directory -Path $source, $workspace | Out-Null + + [IO.File]::WriteAllText( + (Join-Path $source 'synthetic-note.md'), + '# SYNTHETIC_MIGRATION_SENTINEL' + ) + [IO.File]::WriteAllText( + (Join-Path $source 'synthetic-paper.pdf'), + '%PDF-1.4 SYNTHETIC' + ) + + $planRaw = & .\.demo-venv\Scripts\scriptorium.exe migrate plan ` + --source $source ` + --workspace $workspace ` + --batch-id synthetic-ci ` + --json + if ($LASTEXITCODE -ne 0) { throw "Installed migration plan failed" } + $planSerialized = $planRaw -join "`n" + $plan = $planSerialized | ConvertFrom-Json + if ( + $plan.operation -ne 'plan' -or + $plan.status -ne 'planned' -or + $plan.summary.files -ne 2 + ) { + throw "Installed migration plan report changed" + } + if (Test-Path (Join-Path $workspace 'Sources')) { + throw "Migration plan wrote into the workspace" + } + if (Test-Path $env:LOCALAPPDATA) { + throw "Migration plan persisted private state" + } + + $applyRaw = & .\.demo-venv\Scripts\scriptorium.exe migrate apply ` + --source $source ` + --workspace $workspace ` + --batch-id synthetic-ci ` + --json + if ($LASTEXITCODE -ne 0) { throw "Installed migration apply failed" } + $applySerialized = $applyRaw -join "`n" + $apply = $applySerialized | ConvertFrom-Json + if ( + $apply.operation -ne 'apply' -or + $apply.status -ne 'applied' -or + $apply.summary.changed -ne 2 + ) { + throw "Installed migration apply report changed" + } + + $verifyRaw = & .\.demo-venv\Scripts\scriptorium.exe migrate verify ` + --workspace $workspace ` + --batch-id synthetic-ci ` + --json + if ($LASTEXITCODE -ne 0) { throw "Installed migration verify failed" } + $verify = ($verifyRaw -join "`n") | ConvertFrom-Json + if ($verify.operation -ne 'verify' -or $verify.status -ne 'applied') { + throw "Installed migration verify report changed" + } + + $repeatRaw = & .\.demo-venv\Scripts\scriptorium.exe migrate apply ` + --workspace $workspace ` + --batch-id synthetic-ci ` + --json + if ($LASTEXITCODE -ne 0) { throw "Installed migration reapply failed" } + $repeat = ($repeatRaw -join "`n") | ConvertFrom-Json + if ($repeat.status -ne 'unchanged') { + throw "Installed migration reapply was not idempotent" + } + + $rollbackRaw = & .\.demo-venv\Scripts\scriptorium.exe migrate rollback ` + --workspace $workspace ` + --batch-id synthetic-ci ` + --json + if ($LASTEXITCODE -ne 0) { throw "Installed migration rollback failed" } + $rollbackSerialized = $rollbackRaw -join "`n" + $rollback = $rollbackSerialized | ConvertFrom-Json + if ( + $rollback.operation -ne 'rollback' -or + $rollback.status -ne 'rolled-back' -or + $rollback.summary.changed -ne 2 + ) { + throw "Installed migration rollback report changed" + } + + $allReports = @( + $planSerialized, + $applySerialized, + ($verifyRaw -join "`n"), + ($repeatRaw -join "`n"), + $rollbackSerialized + ) -join "`n" + foreach ($privateValue in @( + $root, + $source, + $workspace, + 'synthetic-note.md', + 'synthetic-paper.pdf', + 'SYNTHETIC_MIGRATION_SENTINEL' + )) { + if ($allReports.Contains($privateValue)) { + throw "Migration report exposed a private source value" + } + } + if ( + [IO.File]::ReadAllText((Join-Path $source 'synthetic-note.md')) -ne + '# SYNTHETIC_MIGRATION_SENTINEL' + ) { + throw "Migration modified its source" + } + $remaining = @( + Get-ChildItem ` + (Join-Path $workspace 'Sources\Imported\synthetic-ci') ` + -File ` + -Recurse ` + -ErrorAction SilentlyContinue + ) + if ($remaining.Count -ne 0) { + throw "Migration rollback left owned files" + } - name: Run isolated public pull E2E shell: pwsh run: | & .\.demo-venv\Scripts\python.exe ` .\scriptorium\tests\e2e_pull.py ` - --provenance-root .\Provenance + --provenance-root .\Provenance ` + --spec-root .\scriptorium-spec + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + - name: Run offline Steward to Lectern golden path + shell: pwsh + run: | + & .\.demo-venv\Scripts\python.exe ` + .\scriptorium\tests\e2e_slides.py ` + --steward-root .\steward ` + --spec-root .\scriptorium-spec ` + --lectern-root .\Academic-Slides-Agent if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - name: Run credential-free golden path shell: pwsh @@ -194,4 +398,5 @@ jobs: path: | demo 输出 doctor-report.json + install-lifecycle-report.json retention-days: 14 diff --git a/.gitignore b/.gitignore index 143e54a..8d8f117 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ htmlcov/ build/ dist/ scriptorium-demo/ +.tmp-*/ # Local configuration and credentials .env diff --git a/CHANGELOG.md b/CHANGELOG.md index 7206b87..51bac37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,38 @@ # Changelog +## 0.2.0 (candidate) — 2026-07-22 + +- Add `scriptorium resume`, backed by Provenance's bounded, read-only Context + Capsule. It restores approved project state, explicitly labels auto-applied + low-risk progress, and keeps literature artifacts reference-only. +- Extend the synthetic demo with all four literature-reading artifacts, atomic + idempotent ingestion, reference hints, and privacy assertions. +- Exercise a synthetic software project through Codex and Claude Code workspace + adapters, proving that fresh processes recover the same reviewed project state. +- Require compatible `prov-context` and `prov-ingest-research` commands in Public + Alpha diagnostics, including an actual runtime-version probe. +- Move the candidate compatibility baseline to `scriptorium-spec` 2.3.0 for the + additive `project/1.1` profile field while retaining `project/1.0` compatibility. +- Add pinned `core`, `research`, `slides`, `full`, and `capture` component profiles; + add a preview-first source installer and an independently downloadable Capture + Release asset path without splitting Capture into another repository. +- Add general, engineering, software, and research project templates. Experimental + run/evidence promotion remains on development branches and outside stable-core + acceptance. +- Add the safety-reviewed `scriptorium migrate` CLI to the stable-core candidate for + explicit Markdown/PDF copies: write-free plan, create-if-absent apply, + canonical private state, cross-process verify/reapply recovery, and rollback. + Reports and errors remain aggregate-only and path-free; linked repositories stay + authoritative and are registered rather than copied wholesale. +- Add an offline, synthetic Steward-to-Lectern acceptance path that validates a + two-paper handoff through Lectern's production graph, stops for outline approval + with no persistent pre-approval PPTX, compiles an editable deck, and + scans transferable artifacts for paths, email addresses, and credential shapes. +- Add a temporary clean-environment lifecycle gate that builds local wheels with + package indexes disabled, verifies install/uninstall/reinstall and the synthetic + demo, and exercises the public v0.1.0-to-current version transition. Live Agent, + PowerPoint, external-user, and fresh remote-CI acceptance remain explicit gates. + ## 0.1.0 — 2026-07-20 - Add `scriptorium inventory`, a deterministic, zero-write preview for explicitly @@ -34,7 +67,7 @@ - Add the content-free `scriptorium status` control-plane summary over trusted doctor and pull-preview reports. It exposes allowlisted readiness, freshness, aggregate backlog counts, and fixed review cues without forwarding paths, - identifiers, research content, stderr, or an implicit `--run` authorization. + identifiers, project content, stderr, or an implicit `--run` authorization. It authorizes no suite project/data writes while accurately disclosing external readiness probes. - Report content-free path-selection provenance across `doctor`, `pull`, and @@ -50,8 +83,9 @@ subprocesses cannot reuse the user's real application configuration or caches. - Document verified product-design inspirations, independent implementation boundaries, current capability gaps, and the policy for future third-party reuse. -- Wire the canonical research skill to the aggregate `agent-fill` / `project-resolution` - actions through read-only unresolved inspection, allowlisted scaffold reads, and an +- Wire the canonical project skill (with its compatibility-preserved directory name) + to the aggregate `agent-fill` / `project-resolution` actions through read-only + unresolved inspection, allowlisted scaffold reads, and an atomic credential-checked fill command; candidate and authoritative writes require separate approval and protected paths are never constructed by the skill. - Refuse unresolved-project session summaries and preserve those events for explicit, diff --git a/README.md b/README.md index e47a4e2..44c16cc 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,91 @@ # Scriptorium -> **Public Alpha v0.1.0:** the umbrella repository ships safe initialization for +> **Public Alpha v0.2.0 candidate:** the umbrella repository ships safe initialization for > a real project, one synthetic vertical slice through `scriptorium demo`, the > read-only `scriptorium doctor`, a content-free `scriptorium status` control-plane > summary, a zero-write `scriptorium inventory` preview for explicitly selected > local sources, > and explicit project-scoped Codex and Claude Code skill installers. It also ships -> the accepted on-demand `scriptorium pull` entry through Provenance's -> machine-readable public command. GitHub-hosted clean Windows CI and an isolated -> Windows source-install acceptance run pass; live Agent-host parity and Lectern -> remain outside the credential-free golden path. +> the accepted on-demand `scriptorium pull` entry and bounded, read-only +> `scriptorium resume` Context Capsule through Provenance's machine-readable +> public commands. A safety-reviewed `scriptorium migrate` boundary copies only +> explicitly selected Markdown/PDF files as part of this stable-core candidate. The previously published baseline +> has GitHub-hosted Windows evidence; this worktree also has local core, research-profile, +> and Capture-only installation evidence plus a synthetic software-project recovery check +> through both Codex and Claude Code workspace adapters. Fresh remote CI, live model login, +> and real-provider slide generation remain release gates. + +Scriptorium is a local-first, model-neutral project context and continuity suite for +long-running work. This repository is its thin product entry point: it coordinates +independently useful components through public commands and versioned files without +importing their internals or taking ownership of the user's project data. Research is +the flagship reference workflow, not the product boundary; the same core also supports +engineering maintenance and personal software projects. + +[中文说明](README.zh.md) · [架构、使用与分层验收](docs/architecture-and-acceptance.zh-CN.md) · [中文产品案例](docs/case-study.zh-CN.md) · [Showcase evidence](docs/showcase/README.zh-CN.md) · [Contract source of truth](https://github.com/scriptorium-suite/scriptorium-spec) · [Design inspirations](ACKNOWLEDGEMENTS.md) -Scriptorium is a local-first, agent-native research workflow suite. This repository -is its thin control plane: it coordinates independently useful components through -public commands and versioned files without importing their internals or owning -their research data. +![Scriptorium Public Alpha synthetic golden-path evidence](docs/showcase/demo-poster.svg) -[中文说明](README.zh.md) · [中文产品案例](docs/case-study.zh-CN.md) · [Showcase evidence](docs/showcase/README.zh-CN.md) · [Contract source of truth](https://github.com/scriptorium-suite/scriptorium-spec) · [Design inspirations](ACKNOWLEDGEMENTS.md) +## Product profiles and independent components -![Scriptorium Public Alpha synthetic golden-path evidence](docs/showcase/demo-poster.svg) +`scriptorium components --profile ` shows the pinned, machine-readable component +set for `core`, `research`, `slides`, `full`, or `capture`. `core` contains the shared +Spec contracts and Provenance context engine; `research` adds Steward; `slides` adds +Lectern. Each product keeps its own repository and public CLI. + +`scriptorium install --target ` is a zero-write preview. Add +`--run` only after reviewing the exact repositories, versions, commit IDs, destinations, +and network action. The installer refuses non-empty unowned targets and never deletes a +partial checkout after failure; it leaves a local ownership marker for inspection. + +Capture is not a separate repository. It remains the optional `01-capture` browser +exporter owned by Provenance, while its deterministic ZIP is designed to be attached to +a tagged Provenance Release for users who need only conversation export. Until that +asset is actually published, the `capture` profile reports `action-required` and refuses +to pretend that an independent download is available. + +The stable-core candidate deliberately stops at project intake, session closeout, +human approval, and context recovery. Experimental run/evidence promotion work remains +on development branches and is not included in the component catalog or stable-core +acceptance claim. + +## Short Windows path + +After cloning this entry repository and running `uv sync --locked`, inspect the exact +component plan before allowing any network or filesystem writes: + +```powershell +uv run scriptorium components --profile core +uv run scriptorium install core --target D:\Scriptorium\components +uv run scriptorium install core --target D:\Scriptorium\components --run +. D:\Scriptorium\components\scriptorium-env.ps1 + +uv run scriptorium init ` + --workspace D:\Projects\my-project ` + --provenance-home D:\Projects\my-project-memory ` + --project-id my-project ` + --title "My long-running project" ` + --template software ` + --host codex ` + --run +uv run scriptorium host install codex + +# After completing one Agent session in this workspace: +uv run scriptorium pull --project my-project --json +uv run scriptorium pull --project my-project --run --json +# Complete any reported Agent fill or Approvals.md review, then pull again. +uv run scriptorium resume --project my-project +``` + +Use the `research` profile to add Steward. Use `capture` with a verified local Release +ZIP when only the browser exporter is needed; it does not clone Provenance. Components +remain independently runnable from their own repositories. ## What works now -`scriptorium init` previews or creates the minimal structure for a real research -project: the suite config, separate Markdown workspace and Provenance data-root -directories, and one valid `project/1.0` note. Preview is the default and `--run` +`scriptorium init` previews or creates the minimal structure for a real project: +the suite config, separate Markdown workspace and Provenance data-root +directories, and one valid `project/1.1` note (`project/1.0` remains readable). Preview is the default and `--run` is required to write. Existing files are never rewritten. Selecting a host records that choice in config; it does not install the host adapter, a model, or a hook, and init does not request network access or read provider credentials. After init, @@ -32,6 +93,17 @@ init does not request network access or read provider credentials. After init, selection from the suite config when higher-precedence CLI flags or environment variables are absent. +The CLI defaults to the `general` project template. `--template engineering`, +`--template software`, and `--template research` provide different human-readable +project briefs over the same stable project identity and state contract. + +Existing work enters through explicit routes instead of one unsafe whole-disk import: +register an existing code or engineering repository with `init --linked-repo` and keep +that repository authoritative; inventory and copy only selected Markdown/PDF documents; +bring local Agent logs or a Capture export through Provenance; and use Steward only when +Zotero or literature governance is needed. Scriptorium does not copy an entire repository, +discover personal folders, or ingest credentials merely because a project was linked. + `doctor`, `status`, and `pull` report whether each root came from the CLI, environment, suite config, or auto-discovery. `status` and `pull` reports do not echo selected paths; `doctor` is a detailed local diagnostic and its report @@ -47,9 +119,11 @@ AI4Science literature workflow through the real public interfaces: 1. validate a synthetic `library-kb/1.1` with `scriptorium-spec`; 2. call Steward to scope two papers and assemble a review from a recorded agent draft; 3. call Provenance to ingest the library and Markdown project; -4. build and query the local search index; -5. verify portfolio, project context, and literature search through Provenance MCP; -6. write the human-readable artifacts and `demo-report.json`. +4. validate and atomically ingest four synthetic literature-reading artifacts; +5. build and query the local search index; +6. verify portfolio, bounded Context Capsule, reference-only literature hints, + and literature search through Provenance MCP; +7. write the human-readable artifacts and `demo-report.json`. The run needs no API key, Zotero, Obsidian, browser extension, or agent login. Once the source checkouts are installed, this demo path is designed to operate @@ -77,11 +151,19 @@ its fill in-session, then another pull applies the low-risk timeline and stages high-value claims in `Approvals.md`. A user tick is still required before a later pull commits those claims. +`scriptorium resume` is the bounded session-start view. It asks the compatible +`prov-context` runtime for one registered project and accepts only an allowlisted +capsule shape. Approved project fields are separated from auto-applied low-risk +progress; literature and reading artifacts are explicitly reference-only. Raw +conversations, drafts, rejected claims, component stderr, and local paths are not +forwarded. The V0.2 end-to-end fixture runs two consecutive synthetic sessions and +checks that the second session can recover the first session's reviewed state. + `scriptorium status` is the daily content-free control-plane summary. It first rebuilds the Public Alpha readiness result from `doctor`; only when that boundary is ready does it run a `pull` preview. The result contains allowlisted capability states, aggregate workflow counts, and fixed review cues only. It does not forward local -paths, project or session identifiers, research content, component stderr, or raw +paths, project or session identifiers, project content, component stderr, or raw diagnostic details. It never invokes `--run`: `attention` is a normal exit-0 backlog, while incomplete or blocked readiness returns 1 and an untrustworthy report or a trusted pull-preview error returns 2. Neither status nor its preview authorizes @@ -101,6 +183,34 @@ migration occurred. On Windows, selected objects are held through metadata-only bindings for the duration of the preview, so another process cannot rename, delete, or open them for data write until the command finishes. +`scriptorium migrate` is the stable-core copy boundary for explicitly selected +Markdown/PDF files. `plan` is write-free; first `apply` requires the selected +sources; later `apply`, `verify`, and `rollback` recover the batch from only its +workspace and batch identifier. The preview is advisory rather than a persisted +execution snapshot: first `apply` rescans and re-hashes the explicit sources. +Targets are created without overwrite through a +same-directory hard-link publication step. Apply uses a random, exclusively +created `.scriptorium-*.stage` file; only after one file descriptor has completed +copy, hash, flush, and identity checks is that stage sealed into the private +manifest and published as the target's same-file ownership anchor. A crash before +that seal may leave an unclaimed random stage. It is never adopted or deleted +automatically. + +Rollback records a random same-directory quarantine name before each target and +anchor transition, atomically moves without replacement, then re-verifies content +and recorded file identity before deletion. A replacement detectable through +those recorded properties is restored to its original path when possible, +otherwise preserved in quarantine while rollback fails closed. This coordinates +cooperative local processes; it does not claim protection against a malicious +replacement after every link to a file identity has disappeared, because a +filesystem may reuse that identity. Automatic rollback requires Windows no-replace +rename or Linux `renameat2(RENAME_NOREPLACE)`; other platforms fail closed. Private +path manifests use the canonical per-user local state root outside the workspace; +sources may not overlap that private state root, and no arbitrary state-root flag +is exposed. Terminal +and JSON reports contain aggregate counts and states only. This command copies +files—it does not parse, index, approve, or ingest them into Provenance. + ## Source quickstart on Windows Prerequisites: Git and Python 3.11+. Clone the four repositories into one parent @@ -147,6 +257,53 @@ filenames, research text, hashes, sizes, or timestamps. An incomplete or unsafe fails closed with exit code `1`; malformed invocation or an internal boundary failure returns `2` without echoing the sensitive input. +### Copy selected Markdown/PDF files safely + +Use a stable batch identifier. Review the aggregate plan first, then repeat the +same explicit sources for the first apply: + +```powershell +$Legacy = 'D:\Research\Legacy Notes' +$MigrationWorkspace = 'D:\Research\Scriptorium Workspace' +$Batch = 'legacy-notes-001' + +.\.venv\Scripts\scriptorium.exe migrate plan ` + --source $Legacy ` + --workspace $MigrationWorkspace ` + --batch-id $Batch ` + --json + +.\.venv\Scripts\scriptorium.exe migrate apply ` + --source $Legacy ` + --workspace $MigrationWorkspace ` + --batch-id $Batch ` + --json + +# Recovery operations need no source path or saved plan. +.\.venv\Scripts\scriptorium.exe migrate verify ` + --workspace $MigrationWorkspace ` + --batch-id $Batch ` + --json +.\.venv\Scripts\scriptorium.exe migrate apply ` + --workspace $MigrationWorkspace ` + --batch-id $Batch ` + --json +.\.venv\Scripts\scriptorium.exe migrate rollback ` + --workspace $MigrationWorkspace ` + --batch-id $Batch ` + --json +``` + +The second `apply` is an idempotency/recovery check and should report +`unchanged`. The destination filesystem must support hard links; the command +fails closed instead of using an overwrite-capable fallback. Do not remove the +internal `.scriptorium-*.stage` ownership anchors or `.scriptorium-*.rollback` +quarantine entries while a batch is active; a successful rollback removes its +recorded entries. A process crash before a random stage is recorded can leave an +unclaimed stage for manual inspection; Scriptorium will not guess that it owns or +delete that path. Start with synthetic or isolated copies, review the aggregate plan, +and keep the original source authoritative until verification succeeds. + ### Ten-minute path for a real project The following path creates a real Markdown project rather than using the synthetic @@ -298,10 +455,11 @@ Both paths are explicit and local; use `--json` for the stable machine report: ```powershell scriptorium pull --workspace D:\Research\Workspace --provenance-home D:\Research\ProvenanceData scriptorium pull --workspace D:\Research\Workspace --provenance-home D:\Research\ProvenanceData --run +scriptorium resume --provenance-home D:\Research\ProvenanceData --project my-project ``` The paths must already exist. Scriptorium never falls back to the current directory -for research data. A canonical Codex adapter enables the conservative local-log scan +for project data. A canonical Codex adapter enables the conservative local-log scan (registered projects only, recent stable logs, Desktop excluded). A Claude Code skill does not imply that its optional `SessionEnd` enqueue hook was installed or live-tested; that capture path remains a separate, explicit user configuration. `--project` narrows @@ -311,8 +469,9 @@ If the report includes `project-resolution`, the affected events remain in prote inflight state. Scriptorium will not create a summary, timeline, or draft with an unresolved project. Approve or add the correct `project_id` / `linked_repo` mapping in the Markdown workspace, then rerun pull; the same events resume without being retired. -The canonical research skill inspects unresolved items through the read-only, path- -suppressed `prov-sync-unresolved` entry. When `agent-fill` appears, it reads allowlisted +The canonical project skill (whose directory keeps the legacy `scriptorium-research` +name) inspects unresolved items through the read-only, path-suppressed +`prov-sync-unresolved` entry. When `agent-fill` appears, it reads allowlisted sanitized scaffolds through `prov-sync-pending` and submits approved candidate fills only through `prov-sync-fill`; it never constructs a protected path or writes `fill.json` directly. Fill submission and the later authoritative `--run` require separate approval. @@ -320,7 +479,7 @@ directly. Fill submission and the later authoritative `--run` require separate a Exit `0` includes a successful preview/run and expected `action-required` backlog, `1` means a safe block or partial component pass, and `2` means the entry could not form a trustworthy report. The pull report is deliberately aggregate-only: raw component output, -local paths, session identifiers, and research content are suppressed at the entry boundary. +local paths, session identifiers, and project content are suppressed at the entry boundary. ## Readiness diagnostics @@ -343,8 +502,10 @@ browser-extension permissions, GUI launch, workspace writes, and live network behavior remain explicitly untested. Detecting an application or command is not reported as a successful live integration or provider check. Public Alpha workspace evidence requires at least one `Projects/*.md` note with complete `project/1.x` -frontmatter; an arbitrary repository README is not treated as a research workspace. +frontmatter; an arbitrary repository README is not treated as a Scriptorium workspace. `entry.pull` passes only when the compatible machine-readable capability probe succeeds. +Public Alpha also requires compatible `prov-ingest-research` and `prov-context` +commands, and probes the actual Context Capsule runtime version. Codex provides the first executable session-capture path; a Claude-only installation remains a manual readiness item until its opt-in `SessionEnd` hook is live-verified. @@ -379,7 +540,7 @@ parse the legacy human-readable Provenance status output. scriptorium-demo/ ├── fixtures/ # explicitly synthetic inputs ├── workspace/ -│ ├── Projects/ # project/1.0 Markdown +│ ├── Projects/ # project/1.x Markdown │ ├── Reviews/ # Steward-assembled review │ └── Reports/ # Provenance search + MCP evidence ├── provenance/ @@ -397,12 +558,12 @@ the current report. ## Compatibility baseline -The first golden path intentionally locks exact source versions as its coordinated -Public Alpha release targets: +The current V0.2 candidate intentionally locks exact source versions as its +coordinated compatibility baseline: -- `scriptorium-spec` 2.2.0 +- `scriptorium-spec` 2.3.0 - Steward 0.2.0 -- Provenance 0.17.0 +- Provenance 0.18.0 The demo and CI workflows continue to pin exact component commits. A range-based compatibility policy is intentionally deferred until external Alpha usage provides @@ -410,12 +571,12 @@ evidence for safe ranges. ## Next product increments -1. add a compact project context-capsule/resume entry over Provenance MCP, distinct - from the content-free control-plane `status`; -2. add an explicitly reviewed, adapter-specific migration manifest and apply path; -3. add schema-driven cross-repository E2E for Lectern handoff; -4. verify live Claude Code `SessionEnd` parity with the Codex capture path; -5. run an external-user Alpha and use the evidence to shape packaging and +1. run the current migration, install lifecycle, and slide handoff gates on fresh + remote Windows CI and in an isolated user environment before promotion; +2. commit the stable Spec and Provenance candidates, backfill their final commit IDs + into the component catalog, and rerun clean install from those exact revisions; +3. verify live Codex and Claude Code sessions with the same synthetic project; +4. run an external-user Alpha and use the evidence to shape packaging and compatibility ranges. Apache-2.0. No telemetry. diff --git a/README.zh.md b/README.zh.md index 7a018a0..023b342 100644 --- a/README.zh.md +++ b/README.zh.md @@ -1,25 +1,83 @@ # Scriptorium -> **Public Alpha v0.1.0:** 当前 umbrella 仓库已交付真实项目的安全初始化、一个 +Scriptorium 是面向长期复杂工作的本地优先、模型中立的项目上下文与协作记忆产品。 +本仓库是统一产品入口,通过公开命令和版本化文件协调彼此独立的组件,不导入组件内部 +实现,也不取得用户项目资料的所有权。科研是要求最高的旗舰示例,而不是产品边界; +同一核心也支持工程系统维护和个人软件开发项目。 + +`scriptorium components --profile <名称>` 可以查看 `core`、`research`、`slides`、 +`full` 或 `capture` 对应的固定组件、版本和提交。`scriptorium install <名称> +--target <目录>` 默认只预览,用户明确增加 `--run` 后才会联网安装。Capture 不是 +独立仓库,而是 Provenance 维护的可选浏览器导出器;正式发布时通过单独 Release ZIP +交付。附件尚未实际发布前,命令会明确提示不可用,不会伪装成已经完成。 + +`scriptorium init` 默认生成通用项目模板,也支持 `--template engineering`、 +`--template software` 和 `--template research`。这些模板共享同一套项目身份、状态、 +会话收尾和上下文恢复机制。 + +已有项目按资料类型显式接入,而不是整盘扫描或整仓复制:代码和工程仓库通过 +`init --linked-repo` 登记并继续作为权威来源;用户挑选的 Markdown/PDF 才进入 +`inventory -> migrate`;本地 Agent 日志或 Capture 导出由 Provenance 接入;只有需要 +Zotero 和文献治理时才增加 Steward。仅仅登记仓库不会复制其中的源码、凭据或个人目录。 + +> **Public Alpha v0.2.0 候选版:** 当前 umbrella 仓库已交付真实项目的安全初始化、一个 > 通过 `scriptorium demo` 跑通的合成纵向切片、只读诊断入口 `scriptorium doctor`、 > 不暴露内容的 `scriptorium status` 控制面状态摘要、仅盘点显式本地来源且零写入的 > `scriptorium inventory` 预览,以及显式、项目级的 Codex / -> Claude Code 技能安装器。已确认的按需入口 `scriptorium pull` 也已通过 Provenance -> 的机器可读公共命令交付。GitHub-hosted 干净 Windows CI 与隔离的 Windows 源码 -> 安装验收已通过;Agent host 实时对等路径与 Lectern 仍不属于无凭据黄金路径。 +> Claude Code 技能安装器。已确认的按需入口 `scriptorium pull` 与有边界、只读的 +> `scriptorium resume` Context Capsule 均通过 Provenance 的机器可读公共命令交付。 +> 经过安全复审的 `scriptorium migrate` 已纳入稳定核心候选版,只复制用户显式选择的 +> Markdown/PDF,不会把整座代码仓库或个人目录自动搬入工作区。 +> 已发布基线具有 GitHub-hosted Windows 证据;当前工作树也已在本地通过 core、 +> research 组合与 Capture-only 安装,以及软件项目经 Codex / Claude Code 两种工作区 +> 适配器恢复相同上下文的合成验收。重新运行远端 CI、真实模型登录和真实 provider +> 幻灯片生成仍是发布门槛。 -Scriptorium 是面向科研工作者的本地优先、Agent 原生研究工作流套件。本仓库是 -它的薄控制面:通过公开 CLI 与版本化文件编排可独立使用的组件,不导入组件内部 -模块,也不成为论文库、项目笔记或科研记忆的新数据主库。 +稳定核心候选版有意停在资料接入、会话收尾、人工审批和上下文恢复。实验运行、 +论断—证据晋升等增强能力继续保存在开发分支,不进入当前组件目录,也不计入稳定核心 +的可用性声明。 -[English](README.md) · [中文产品案例](docs/case-study.zh-CN.md) · [展示与证据](docs/showcase/README.zh-CN.md) · [契约单一事实源](https://github.com/scriptorium-suite/scriptorium-spec) · [设计借鉴与致谢](ACKNOWLEDGEMENTS.zh.md) +## Windows 最短使用路径 + +克隆本入口仓库并执行 `uv sync --locked` 后,先查看将要安装的固定组件和提交;只有 +确认无误后才增加 `--run`: + +```powershell +uv run scriptorium components --profile core +uv run scriptorium install core --target D:\Scriptorium\components +uv run scriptorium install core --target D:\Scriptorium\components --run +. D:\Scriptorium\components\scriptorium-env.ps1 + +uv run scriptorium init ` + --workspace D:\Projects\my-project ` + --provenance-home D:\Projects\my-project-memory ` + --project-id my-project ` + --title "我的长期项目" ` + --template software ` + --host codex ` + --run +uv run scriptorium host install codex + +# 在该工作区完成一次 Agent 协作后: +uv run scriptorium pull --project my-project --json +uv run scriptorium pull --project my-project --run --json +# 按报告完成 Agent 填充或 Approvals.md 审阅,再次 pull 后继续: +uv run scriptorium resume --project my-project +``` + +需要文献治理时选择 `research` 组合,它只在核心上增加 Steward。只需要浏览器导出器时, +使用已校验的 Capture Release ZIP;该路径不会克隆 Provenance。Provenance、Steward 和 +Lectern 仍可分别从各自仓库独立使用。 + +[English](README.md) · [架构、使用与分层验收](docs/architecture-and-acceptance.zh-CN.md) · [中文产品案例](docs/case-study.zh-CN.md) · [展示与证据](docs/showcase/README.zh-CN.md) · [契约单一事实源](https://github.com/scriptorium-suite/scriptorium-spec) · [设计借鉴与致谢](ACKNOWLEDGEMENTS.zh.md) ![Scriptorium Public Alpha 合成黄金路径证据](docs/showcase/demo-poster.svg) ## 当前真正可用的内容 -`scriptorium init` 可以预览或创建真实科研项目的最小结构:套件配置、彼此分离的 -Markdown workspace 与 Provenance 数据根目录,以及一份有效的 `project/1.0` 项目笔记。 +`scriptorium init` 可以预览或创建真实长期项目的最小结构:套件配置、彼此分离的 +Markdown workspace 与 Provenance 数据根目录,以及一份有效的 `project/1.1` 项目笔记 +(旧的 `project/1.0` 仍可读取)。 默认只预览,必须显式增加 `--run` 才会写入;已有文件不会被重写。选择 host 只会把 该选择记录到配置中,不会安装 host adapter、模型或 hook;init 不申请网络动作,也不 读取 provider 凭据。初始化后,如果没有更高优先级的 CLI 参数或环境变量, @@ -39,9 +97,11 @@ workspace、数据根与项目选择。 1. 使用 `scriptorium-spec` 校验合成 `library-kb/1.1`; 2. 调用 Steward 选出两篇主题文献,并从录制的 Agent 草稿组装综述; 3. 调用 Provenance 摄取文献库与 Markdown 项目; -4. 构建和查询本地全文索引; -5. 通过 Provenance MCP 验证项目组合、当前上下文和文献检索; -6. 生成可阅读成果与机器可读的 `demo-report.json`。 +4. 校验并原子摄取四类合成文献阅读工件; +5. 构建和查询本地全文索引; +6. 通过 Provenance MCP 验证项目组合、有边界的 Context Capsule、仅供参考的 + 文献线索和文献检索; +7. 生成可阅读成果与机器可读的 `demo-report.json`。 该流程不需要 API Key、Zotero、Obsidian、浏览器扩展或 Agent 登录。源码安装 完成后,此 demo 路径被设计为不申请网络动作;它不会调用在线模型,Agent 产出是 @@ -63,15 +123,21 @@ doctor 才会通过 host adapter 检查。它会真实探测 `prov-sync-pull --c 才会提交低风险 timeline,并把高价值声明放入 `Approvals.md`。这些声明仍必须由用户 勾选,之后的 pull 才会提交。 +`scriptorium resume` 是会话开始时读取的精炼视图。它只接受兼容 `prov-context` +运行时返回的白名单 Capsule:把已批准项目状态、自动写入的低风险进度和仅供参考的 +文献/阅读工件明确分开;不会透传原始对话、草稿、已拒绝声明、组件 stderr 或本地路径。 +V0.2 端到端夹具会连续运行两个合成 Agent 会话,并验证第二次能恢复第一次经过审阅的 +项目状态。 + `scriptorium status` 是日常使用且不暴露内容的控制面状态摘要。它先重建 `doctor` 的 Public Alpha 就绪结论;只有该边界就绪时,才执行一次 `pull` 预览。结果只包含 白名单化的能力状态、工作流聚合计数和固定审阅提示,不会透传本地路径、项目或会话 -标识、科研正文、组件 stderr 或原始诊断细节,也绝不会调用 `--run`。`attention` +标识、项目正文、组件 stderr 或原始诊断细节,也绝不会调用 `--run`。`attention` 是正常的 exit-0 待办;未就绪或安全阻断返回 1,可信 pull 预览报告错误或入口无法 形成可信报告时返回 2。status 与其中的预览都不会授权套件写入项目或数据;就绪检查 仍会调用外部版本/能力探针,其操作系统级副作用尚未被观测。 -`scriptorium inventory` 是已有科研资料进入套件前的安全盘点边界。它只扫描用户显式 +`scriptorium inventory` 是已有项目资料进入套件前的安全盘点边界。它只扫描用户显式 传入的 Markdown/PDF 来源、AI 对话导出或 Zotero 导出;只读取文件系统元数据与后缀, 不会打开正文或压缩包、自动发现个人目录、写入迁移计划、调用组件、申请网络动作或 调用模型。默认报告不含路径和文件名,只给出候选总数以及 workspace、文献原址引用、 @@ -79,6 +145,26 @@ Provenance 导入审阅、Steward 审阅四类路由。该预览不验证文件 资料,也不会声称迁移已经发生。在 Windows 上,命令会在预览期间以仅元数据句柄绑定 选中对象,因此其他进程需等命令结束后才能重命名、删除或以数据写权限打开这些对象。 +`scriptorium migrate` 是稳定核心候选版的显式 Markdown/PDF 复制边界。`plan` +零写入;首次 `apply` 必须再次给出已核对的来源;后续 `apply`、`verify` 和 +`rollback` 只凭 workspace 与批次标识恢复。预览只是 advisory,不是持久化执行快照; +首次 `apply` 会重新扫描并哈希显式来源。目标文件通过同目录 hard link 以 +create-if-absent 方式发布,绝不覆盖。`apply` 使用随机、独占创建的 +`.scriptorium-*.stage`:同一个文件句柄完成复制、哈希、刷盘和文件身份检查后,才把 +stage 与身份封入私有 manifest,并发布为目标的同文件所有权锚点。若进程在封存前崩溃, +可能留下未认领的随机 stage;系统不会猜测归属,也不会自动删除它。 + +回滚会先持久化随机的同目录 quarantine 名,再用不覆盖的原子移动分别隔离目标和锚点, +移动后重新检查内容与已记录文件身份,确认归属后才删除。能由这些记录属性识别的外部 +替代文件会尽量原子恢复到原路径;恢复受阻时保留在 quarantine 并 fail closed。 +该机制只协调单用户下的协作进程;最后一个链接消失后文件系统可能复用 inode/file ID, +因此不宣称抵御恶意本地进程主动替换。自动回滚要求 Windows no-replace rename 或 +Linux `renameat2(RENAME_NOREPLACE)`;其他平台不降级执行。 +含绝对路径的私有 manifest 固定存放在 workspace 外的用户级 canonical 本地状态根, +迁移来源不得与该私有状态根重叠,CLI 不提供任意 `state-root` 参数。终端和 JSON +报告只含聚合计数与状态。迁移只复制 +文件,不会解析、索引、批准科研内容,也不会自动导入 Provenance。 + ## Windows 源码快速体验 前置条件:Git 与 Python 3.11+。先把四个仓库克隆到同一个父目录,使源码发现 @@ -118,10 +204,53 @@ python -m venv .venv ``` 该命令只做分类,始终保持 preview 模式。默认终端输出与 `--json` 都只包含计数和固定 -路由标签,不含本地路径、文件名、科研正文、哈希、大小或时间。扫描不完整或来源不安全 +路由标签,不含本地路径、文件名、资料正文、哈希、大小或时间。扫描不完整或来源不安全 时会 fail closed 并返回退出码 `1`;参数错误或入口边界内部失败返回 `2`,且不会回显 敏感输入。 +### 安全复制显式选择的 Markdown/PDF + +先给批次一个稳定标识并检查聚合预览;首次执行时再次传入相同来源: + +```powershell +$Legacy = 'D:\Research\Legacy Notes' +$MigrationWorkspace = 'D:\Research\Scriptorium Workspace' +$Batch = 'legacy-notes-001' + +.\.venv\Scripts\scriptorium.exe migrate plan ` + --source $Legacy ` + --workspace $MigrationWorkspace ` + --batch-id $Batch ` + --json + +.\.venv\Scripts\scriptorium.exe migrate apply ` + --source $Legacy ` + --workspace $MigrationWorkspace ` + --batch-id $Batch ` + --json + +# 恢复操作不再需要来源路径或旧 plan。 +.\.venv\Scripts\scriptorium.exe migrate verify ` + --workspace $MigrationWorkspace ` + --batch-id $Batch ` + --json +.\.venv\Scripts\scriptorium.exe migrate apply ` + --workspace $MigrationWorkspace ` + --batch-id $Batch ` + --json +.\.venv\Scripts\scriptorium.exe migrate rollback ` + --workspace $MigrationWorkspace ` + --batch-id $Batch ` + --json +``` + +第二次 `apply` 是幂等/恢复检查,应返回 `unchanged`。目标文件系统必须支持 hard +link;不支持时命令会 fail closed,不会降级到可能覆盖目标的方式。批次生效期间不要 +删除内部 `.scriptorium-*.stage` 所有权锚点或 `.scriptorium-*.rollback` quarantine; +成功回滚会清理已登记条目。若进程在随机 stage 登记前崩溃,未认领 stage 只供人工核查, +Scriptorium 不会认领或自动删除。首次使用时应先在合成资料或隔离副本上验证,审阅 +聚合计划,并在 `verify` 成功前继续把原始来源视为权威副本。 + ### 真实项目的 10 分钟路径 以下命令创建真实 Markdown 项目,而不是使用合成 demo。workspace 与 Provenance @@ -262,9 +391,10 @@ scriptorium host install claude-code --workspace D:\Research\MyProject ```powershell scriptorium pull --workspace D:\Research\Workspace --provenance-home D:\Research\ProvenanceData scriptorium pull --workspace D:\Research\Workspace --provenance-home D:\Research\ProvenanceData --run +scriptorium resume --provenance-home D:\Research\ProvenanceData --project my-project ``` -两个路径都必须已经存在;Scriptorium 永不把当前目录隐式当作科研数据根。已登记的 +两个路径都必须已经存在;Scriptorium 永不把当前目录隐式当作项目数据根。已登记的 canonical Codex adapter 会启用保守的本地日志扫描(只收已登记项目、最近且稳定的日志、 排除 Desktop)。安装 Claude Code skill 不等于其可选 `SessionEnd` enqueue hook 已安装或 完成 live 验证;该捕获路径仍需用户单独显式配置。`--project` 只收窄 Codex 发现范围, @@ -273,14 +403,15 @@ workspace 摄取和已有同步队列仍是 workspace-wide。 如果报告出现 `project-resolution`,对应事件会继续保留在受保护的 inflight 状态; Scriptorium 不会为未解析项目生成摘要、timeline 或 draft。请在 Markdown workspace 中 批准或补充正确的 `project_id` / `linked_repo` 映射,再次 pull 后会继续原事件而不是将其 -退休。canonical research skill 通过默认隐藏路径的只读 `prov-sync-unresolved` 检查未解析 -事件。出现 `agent-fill` 时,它只通过 `prov-sync-pending` 读取白名单化的已净化 scaffold, +退休。canonical 项目 Skill(目录为兼容旧版本保留 `scriptorium-research` 名称)通过 +默认隐藏路径的只读 `prov-sync-unresolved` 检查未解析事件。出现 `agent-fill` 时,它只通过 +`prov-sync-pending` 读取白名单化的已净化 scaffold, 并只通过 `prov-sync-fill` 提交用户批准的候选 fill;不会拼接受保护路径或直接写 `fill.json`。提交 fill 与后续执行权威 `--run` 需要分别授权。 退出码 `0` 包括成功预览/执行和正常的 `action-required` 待办;`1` 表示安全阻断或部分 阶段失败;`2` 表示入口无法形成可信报告。pull 报告按设计只包含聚合信息:组件原始输出、 -本地路径、会话标识和科研正文都会在入口边界被抑制。 +本地路径、会话标识和项目正文都会在入口边界被抑制。 ## 就绪度诊断 @@ -300,8 +431,10 @@ scriptorium doctor --json ` 扩展缺失,只会降级对应能力。Agent 登录、浏览器扩展权限、GUI 启动、workspace 写入 与真实网络行为均明确标为未测试。检测到应用或命令,不等于对应 live integration、 provider 或真实产出已经验证。Public Alpha 的 workspace 证据至少要求一个带完整 -`project/1.x` frontmatter 的 `Projects/*.md` 笔记;普通仓库 README 不会被误判为科研 -workspace。`entry.pull` 只有在兼容的机器可读能力探针通过时才通过。Codex 是当前第一条 +`project/1.x` frontmatter 的 `Projects/*.md` 笔记;普通仓库 README 不会被误判为 +Scriptorium workspace。`entry.pull` 只有在兼容的机器可读能力探针通过时才通过。Public Alpha +还要求兼容的 `prov-ingest-research` 与 `prov-context` 命令,并探测真实 Context Capsule +运行时版本。Codex 是当前第一条 可执行会话捕获路径;仅安装 Claude 的配置在其 opt-in `SessionEnd` hook 完成 live 验证前, 仍会被诚实标为人工就绪项。 @@ -334,7 +467,7 @@ status 输出。 scriptorium-demo/ ├── fixtures/ # 明确标注的合成输入 ├── workspace/ -│ ├── Projects/ # project/1.0 Markdown +│ ├── Projects/ # project/1.x Markdown │ ├── Reviews/ # Steward 组装的综述 │ └── Reports/ # Provenance 检索与 MCP 证据 ├── provenance/ @@ -350,21 +483,18 @@ scriptorium-demo/ ## 当前兼容基线 -- `scriptorium-spec` 2.2.0 +- `scriptorium-spec` 2.3.0 - Steward 0.2.0 -- Provenance 0.17.0 +- Provenance 0.18.0 -首个 golden path 将以上版本作为协同 Public Alpha 的发布目标,并在 demo 与 CI 中 -继续固定精确组件提交。版本范围兼容策略暂不靠猜测定义,而是在获得外部 Alpha -使用证据后再确定。 +当前 V0.2 候选版将以上版本作为协同兼容基线,并在 demo 与 CI 中继续固定精确组件 +提交。版本范围兼容策略暂不靠猜测定义,而是在获得外部 Alpha 使用证据后再确定。 ## 紧邻的产品增量 -1. 在 Provenance MCP 之上增加精炼的项目 context-capsule/resume 入口,并与不暴露 - 内容的控制面 `status` 明确区分; -2. 增加需要显式人审的适配器级迁移清单与执行路径; -3. 补齐 Lectern handoff 的 schema 驱动跨仓 E2E; -4. 验证 Claude Code `SessionEnd` 与 Codex 捕获路径的实时对等性; -5. 开展外部用户 Alpha,用实测结果决定安装打包与兼容版本范围。 +1. 在全新的远端 Windows CI 与隔离用户环境中复跑当前迁移、安装生命周期和幻灯片交接门禁后,再升级其发布状态; +2. 提交稳定 Spec 与 Provenance 候选,把最终提交号回填到组件目录,再按这些精确版本复跑干净安装; +3. 用同一合成项目验证真实 Codex 与 Claude Code 会话; +4. 开展外部用户 Alpha,用实测结果决定安装打包与兼容版本范围。 Apache-2.0,无遥测。 diff --git a/docs/architecture-and-acceptance.zh-CN.md b/docs/architecture-and-acceptance.zh-CN.md new file mode 100644 index 0000000..2dfc253 --- /dev/null +++ b/docs/architecture-and-acceptance.zh-CN.md @@ -0,0 +1,460 @@ +# Scriptorium 架构、使用与分层验收说明 + +> 文档状态:稳定核心候选版验收说明,依据 2026-08-10 的隔离工作树编写。 +> +> 当前工作树尚未提交或推送,不等同于已经发布。实验运行与论断—证据晋升保留在开发分支,不阻挡稳定核心收口。 + +## 1. 一句话理解这个产品 + +**Scriptorium 是本地优先、模型中立的长期项目控制层。它连接使用者、AI Agent、本地资料和可选专业工具,把项目目标、会话、决定、进展和交付物沉淀为可恢复、可审查、可追溯的项目状态。** + +通俗地说,它不是替用户自动“做完项目”,也不是另一个聊天机器人。它负责让长期工作不断线: + +- 新一轮 AI 协作能知道项目做到哪里; +- 资料、会话、判断和交付物能找到来源; +- AI 建议与用户已经确认的决定不会混在一起; +- 关键写入先预览、再确认,必要时可以回滚; +- Codex、Claude Code 或其他 Agent 可以更换,项目状态仍留在本地文件和版本化契约中。 + +稳定核心覆盖通用、工程、软件与科研四类模板,共用同一套项目身份、会话收尾、人工审批 +和上下文恢复机制。实验运行、论断—证据审核与执行器接线属于增强方向,继续在开发分支 +验证,不进入本轮稳定发布口径。 + +## 2. 各组件分别做什么 + +| 组件 | 面向用户的职责 | 不负责什么 | +|---|---|---| +| **Scriptorium(umbrella)** | 统一入口;初始化项目;检查环境;盘点显式选中的资料;触发会话同步;读取恢复上下文;安装项目级 Agent Skill | 不直接拥有 Zotero、记忆库或 PPT 编译器;不在后台偷偷运行 | +| **scriptorium-spec** | 定义各组件交换的 JSON Schema 和约定,是跨仓库数据契约的唯一事实来源 | 没有运行时产品,也不保存用户研究数据 | +| **Steward** | 可选科研扩展;管理文献资料、阅读产物与 `handoff/1.x` 交接包 | 不作为通用项目核心依赖;不替用户确认科学结论 | +| **Provenance** | 保存会话、项目状态、审批候选和检索索引;生成有边界的 Context Capsule | 不替 Agent 做开放式推理;不会把 Agent 建议自动升级为正式决定 | +| **外部 Agent** | 阅读项目上下文,辅助梳理问题、资料、分析和待审候选 | 不是项目事实的最终裁决者 | +| **外部专业工具** | 运行代码、计算、文献或领域工作 | 稳定核心不强制接管其执行过程 | +| **Lectern** | 消费论文或 `handoff/1.x`,先生成可审阅提纲,再编译为可编辑 `.pptx` | 离线跨仓验收使用 `FakeLLM`,不等同于真实 provider、真实论文或 PowerPoint 人工 UAT | + +Obsidian、Zotero 和 PowerPoint 是可选应用。核心交换面是本地 Markdown 与版本化 JSON;没有这些应用时,项目仍应能够以文件方式工作,只是体验会降级。 + +## 3. 当前整体架构 + +```mermaid +flowchart TD + U["使用者:目标、审批、最终判断"] --> S["Scriptorium 统一入口"] + A["Codex / Claude Code / 其他 Agent"] <--> S + S <--> W["本地 Markdown 工作区"] + S <--> P["Provenance:会话、项目状态、候选审批、Context Capsule"] + S -. "research profile" .-> T["Steward:文献 KB、解析、阅读、综述、handoff"] + T --> C["scriptorium-spec:版本化文件契约"] + P --> C + W --> C + E["代码 / Jupyter / 工程与领域工具"] -. "由用户显式运行并按需接回" .-> W + T --> H["handoff/1.x"] + H --> L["Lectern:提纲审阅 -> 可编辑 PPTX"] +``` + +组件之间的基本原则是:**通过公开命令、只读 MCP 和文件契约协作,不直接调用彼此内部实现。** + +数据边界如下: + +- 默认保存在本机; +- 本地命令不应自行上传资料; +- 联网文献检索只能发送公开检索词、题名、标识符或用户明确批准的材料; +- 使用外部模型或云解析器时,由对应 Agent、Steward 或 Lectern 的显式配置决定,不能把它描述成离线; +- 私有笔记、未发表结果、原始会话、凭据和本地路径不得进入公开测试数据或公开报告。 + +## 4. 整体怎么使用 + +### 4.0 先选择安装组合 + +`core` 只包含 Spec 与 Provenance;`research` 在核心上增加 Steward;`slides` 增加 +Lectern;`capture` 只安装经过校验的浏览器扩展 ZIP,不克隆 Provenance。所有安装默认 +只预览: + +```powershell +scriptorium components --profile core --json +scriptorium install core --target D:\Scriptorium\components --json +scriptorium install core --target D:\Scriptorium\components --run --json +. D:\Scriptorium\components\scriptorium-env.ps1 +``` + +各组件仍保留自己的仓库和命令。统一入口负责给普通用户提供固定版本组合,并不把组件 +合并成一个必须整体下载的单体仓库。 + +### 4.0.1 再跑公开 synthetic golden flow + +在接触真实资料前,先用仓库自带的完全合成数据检查组件接线: + +```powershell +scriptorium demo ` + --output ` + --spec-root ` + --steward-root ` + --provenance-root +``` + +当前 demo 会验证版本化示例、Steward 文献综述、Provenance 项目与研究产物导入、重复导入幂等、Context Capsule、检索和只读 MCP。会话的 `init -> pull -> approval -> resume` 双轮闭环由独立的合成 E2E 测试验证。 + +主 `scriptorium demo` 不读取真实 Zotero、真实 Agent 日志或模型凭据,也不调用 Lectern +和外部实验。另有 `tests/e2e_slides.py` 使用程序生成的两篇合成 PDF,验证 +`Steward -> handoff/1.1 -> Spec -> Lectern -> 提纲审批 -> 可编辑 PPTX`;这条测试同样 +不调用真实 provider。两条链路都只能证明软件按契约协作,**不能证明真实科研结论、 +实时模型路径或外部用户安装已经通过**。 + +### 4.1 首次建立项目 + +先检查组件是否可用: + +```powershell +scriptorium doctor --target public-alpha --json +``` + +然后预览初始化计划。默认不写入: + +```powershell +scriptorium init ` + --workspace ` + --provenance-home ` + --project-id ` + --title "" ` + --host codex ` + --template software ` + --context "<current-project-context>" ` + --json +``` + +确认工作区、数据根目录、项目标识、项目模板和初始上下文都正确后,才增加 `--run`。 +初始化会建立最小 Markdown 工作区和套件配置,不会自动读取 Zotero、模型密钥或整台电脑。 + +### 4.2 盘点已有资料 + +`inventory` 只扫描用户明确给出的 Markdown、PDF、会话导出或 Zotero 导出位置,并输出聚合预览: + +通用项目采用分流接入:已有代码或工程仓库通过 `init --linked-repo` 登记,原仓库继续 +作为权威来源;需要放入 Markdown 工作区的资料只复制用户显式选中的 Markdown/PDF; +Agent 日志和 Capture 导出交给 Provenance;Zotero 与文献资料交给可选的 Steward。 +登记仓库不会触发整仓复制、秘密扫描或自动索引。 + +```powershell +scriptorium inventory --source <selected-file-or-directory> --json +``` + +它当前是**零写入盘点边界**:不复制、不解析、不索引、不迁移。报告用于回答“有多少资料、可以路由到哪里、哪些类型暂不支持”,而不是展示私人正文。 + +### 4.3 迁移资料:稳定核心文档复制流程 + +Markdown/PDF 迁移引擎已经接入公开 `scriptorium migrate` CLI,并纳入本轮稳定核心 +候选。入口只接受用户显式给出的来源, +默认不联网、不调用模型、不解析正文,也不会把副本自动升级为 Provenance 事实。 + +```powershell +# 零写入预览;只输出聚合计数。 +scriptorium migrate plan ` + --source <selected-file-or-directory> ` + --workspace <isolated-workspace> ` + --batch-id <stable-batch-id> ` + --json + +# 新批次首次 apply 必须再次给出来源。 +scriptorium migrate apply ` + --source <selected-file-or-directory> ` + --workspace <isolated-workspace> ` + --batch-id <stable-batch-id> ` + --json + +# 已有批次仅凭 workspace + batch ID 恢复。 +scriptorium migrate verify --workspace <isolated-workspace> --batch-id <stable-batch-id> --json +scriptorium migrate apply --workspace <isolated-workspace> --batch-id <stable-batch-id> --json +scriptorium migrate rollback --workspace <isolated-workspace> --batch-id <stable-batch-id> --json +``` + +其中第二次 `apply` 应返回 `unchanged`。私有 manifest 固定写到 workspace 外的 +canonical 用户级本地状态根,CLI 不允许分叉到任意 `state-root`。`plan` 是零写入 +advisory preview,不是持久化执行快照;首次 `apply` 会重新扫描并哈希显式来源。 +目标文件先以随机 +O_EXCL 名在目标同目录暂存;同一 fd 完成复制、哈希、刷盘与 `fstat` 后,才把随机 +stage 名与稳定文件身份封入 manifest,再以 hard link 原子执行 create-if-absent。 +文件系统不支持时 fail closed,不会降级为可覆盖发布。封存前崩溃可能留下未认领随机 +stage;后续重试不会扫描、认领或自动删除它,因此不会把竞争者文件误当成自己的锚点。 + +回滚采用 `target -> target-quarantined -> anchor -> anchor-quarantined -> deleted` +持久化状态机。每一步先记录 128-bit 随机同目录 quarantine 名,再用 Windows +no-replace rename 或 Linux `renameat2(RENAME_NOREPLACE)` 原子移动,移动后重新验证 +哈希、大小和已记录文件身份;能由这些属性识别的外部替代文件只恢复或保留,不删除。 +该状态机协调单用户下的协作进程;最后一个链接消失后文件系统可能复用 inode/file ID, +因此不宣称抵御恶意本地进程主动替换。平台不支持原子 no-replace move、恢复路径被占用 +或身份不一致时均 fail closed。active batch 期间不得删除已登记的 +`.scriptorium-*.stage` / `.scriptorium-*.rollback`;成功回滚最后清理已登记条目, +未认领 orphan 只供人工核查。终端与 +JSON 报告、参数错误和运行时错误均不回显路径、文件名或正文。 + +当前已由合成测试覆盖显式输入、无覆盖、拒绝链接/重解析点、进程退出释放锁、字节哈希 +一致、随机 stage 抢占保护、封存前 crash 重试、四阶段 quarantine crash 恢复、 +来源丢失后的幂等重跑、可恢复回滚与私有状态外置;还需要把干净 +Windows CI、隔离用户 UAT 和更广泛文件系统兼容结果作为升级候选状态的发布证据。 + +### 4.4 日常会话闭环 + +每次继续项目时,先读取有边界的项目摘要: + +```powershell +scriptorium resume --project <project-id> --json +``` + +`resume` 是只读操作。它返回项目目标、已批准状态、近期进展、下一步和参考资料提示,不把所有历史对话塞进上下文。文献研究产物固定标为 `reference_only`,只能作为导航,不能直接当成已确认的科学结论。 + +Agent 完成本轮工作后,先预览会话同步: + +```powershell +scriptorium pull --project <project-id> --json +``` + +只有在用户明确同意推进同步后才运行: + +```powershell +scriptorium pull --project <project-id> --run --json +``` + +需要注意: + +- 未解析到项目的会话先进入 project resolution,不能生成正式 `session-summary/1.0`; +- Agent 生成的总结填充是候选内容,不能自行写成项目事实; +- 低风险时间线与高价值结论采用不同的审批边界; +- `Approvals.md` 只能由用户审阅和勾选,Agent 不得代替用户批准; +- 重复执行不应重复添加同一事件。 + +### 4.5 文献工作 + +有 Zotero 时,可以由 Steward 对本地文献库做备份、只读盘点和导出;没有 Zotero 时,也可以从显式文件和已经导出的 KB 开始。典型产物包括: + +- `library-kb/1.x`:文献目录; +- `parsed-paper/1.0`:结构化论文; +- `reading-note/1.0`:分阶段阅读笔记; +- `lineage-graph/1.0`:库内引用脉络; +- `review/1.0`:综述产物; +- `handoff/1.x`:交给 Lectern 的论文与元数据包。 + +这些文件应先通过 scriptorium-spec 验证,再导入 Provenance。导入后仍保持参考资料身份,直到研究者把具体论断与原始证据核对并批准。 + +### 4.6 外部实验 + +当前建议这样做: + +1. 从 Context Capsule 和项目笔记中选出一个可证伪问题; +2. 在隔离的 Jupyter、Python 或领域软件环境运行最小实验; +3. 保存代码、输入版本、参数、随机种子、日志和输出文件; +4. 对关键数字做独立复算; +5. 由研究者审阅后,把结论、限制和下一步写回项目。 + +稳定核心没有实验账本或论断—证据晋升承诺;“脚本成功退出”不能被 Scriptorium 自动 +解释为“结论成立”。相关原型与设计保存在开发分支,等核心产品获得真实使用反馈后再决定 +是否进入后续稳定版本。 + +### 4.7 生成汇报 PPT + +Steward 先生成 `handoff/1.x`,Lectern 再按“先提纲、后编译”的方式工作: + +```powershell +lectern outline <handoff-directory-or-pdf> --out outline.json +``` + +研究者检查叙事、页数、图表、引用和保密边界后,明确批准该提纲,再运行: + +```powershell +lectern build --from-outline outline.json --out deck.pptx +``` + +不建议 Agent 使用一条命令直接从来源生成 PPT,因为 one-shot 路径会自动跨过提纲审批。 +套件跨仓合成验收已经证明:审批前输出目录没有 PPTX,批准后生成的标题和原生表格可在 +`python-pptx` 中重新打开、修改、保存并再次打开。真实模型路径与本机 PowerPoint 打开编辑 +仍属于 V0.4 私有 UAT。 + +## 5. 分层验收标准 + +### 5.1 V0.2:证明“下一次协作接得上” + +目标是让通用、工程、软件和科研项目都能把显式资料、两轮会话和项目状态接入同一个 +可恢复闭环;科研产物作为要求更高的旗舰集成样例。 + +公开 synthetic 验收至少应满足: + +1. `init` 支持四类模板,预览零写入,执行后再次运行保持幂等; +2. `linked-repo` 只登记现有仓库;`inventory` 零写入;`migrate` 只复制显式选择的 Markdown/PDF,并可验证和回滚; +3. `doctor` 能识别精确兼容的 Spec、Steward 和 Provenance; +4. 两轮合成 Agent 会话经过 `pull`、项目解析、候选填充和人工审批边界; +5. 第二轮 `resume` 能看到已经接受的进展,而不是原始私聊全文; +6. 四类合成研究产物通过 Schema 验证并导入,逆序重复导入后新增数为零且存储字节稳定; +7. Context Capsule 有固定大小上限,不含本地路径,研究产物均为 `reference_only`; +8. 未解析项目不生成 `session-summary/1.0`; +9. 测试输出和公开 artifact 通过邮箱、凭据、绝对路径与私人关键词扫描; +10. Windows 为阻断平台,Ubuntu 至少保持回归通过。 + +当前判断:V0.2 的四类模板、显式文档迁移、`resume`、研究产物导入、合成 demo 扩展和双会话 E2E 已通过本地 +自动化验证;它们还未形成正式 V0.2 发布。远端 Windows 与 Ubuntu CI、干净 diff、最终 +版本提交和 release tag 仍是发布结论的必要证据。 + +### 5.2 开发分支:保存增强能力,不阻挡核心发布 + +开发分支保留两类已经投入过的增强探索: + +1. **最小实验账本与论断证据** + - 契约和晋升机制仍处于开发状态,不属于稳定核心; + - 能记录运行身份、代码版本、输入、参数、环境、状态、指标和产物哈希; + - 失败运行也保留; + - “运行完成”与“结论被证据支持”是两个独立状态。 +2. **汇报交付闭环** + - Steward 生成合法 `handoff/1.x`; + - Lectern 生成提纲并停在人工审批; + - 批准后生成可打开、可编辑的 `.pptx`; + - 每个关键结论能回到来源或明确标为推测。 + +当前判断:这些内容可以继续作为正在推进的开发线保存,但不进入稳定组件目录、快速开始 +路径或发布完成度。稳定版是否吸收它们,取决于核心产品的真实使用反馈。 + +### 5.3 稳定核心发布:证明“别人能在 Windows 上独立用起来” + +稳定核心的产品验收不是只看单仓测试: + +1. 一台干净 Windows 环境能按文档完成安装、升级和卸载; +2. Codex 与 Claude Code 各完成至少一轮真实但隔离的会话恢复与收尾; +3. Zotero、Obsidian、PowerPoint、Lectern 缺失时给出明确降级,不破坏 Markdown 核心路径; +4. 用户可以选择 research 扩展或只安装 Capture;Provenance 与 Steward 可独立运行; +5. 所有默认动作本地优先、预览优先、无后台常驻、无隐式网络; +6. 至少一名未参与开发的目标用户,在不依赖开发者代操作的情况下完成: + - 建立或迁入一个项目; + - 恢复上下文; + - 完成一轮项目推进; + - 关闭会话并在下一轮恢复; + - 看到一份可审阅的项目状态; +7. 记录任务完成率、首次成功耗时、阻塞点、误操作和用户是否理解审批边界; +8. 发布包、CI artifact、截图和文档经过隐私与凭据扫描。 + +当前判断:本地 Windows 已通过 wheel 干净安装、卸载、重装、core/research/Capture-only +选择安装、独立组件启动和双适配器合成恢复。最终提交号回填、远端 CI、真实双 Agent 登录 +与外部用户 Alpha 仍未完成,因此当前仍是候选版而不是已发布产品。 + +## 6. 验收数据如何分级 + +| 等级 | 可以包含什么 | 可以放到哪里 | 规则 | +|---|---|---|---| +| **S0:公开合成数据** | 明确标注 `[SYNTHETIC]` 的虚构论文、项目、会话、指标和 PPT 内容 | 仓库、CI、公开 demo、截图 | 不得从真实研究材料改写后假装合成 | +| **S1:生成的无害化数据** | 由 S0 运行生成的聚合报告、哈希、状态、匿名 ID、可公开产物 | 隔离测试目录;通过扫描后可作为 CI artifact | 不含邮箱、用户名、绝对路径、密钥、原始会话或真实研究内容 | +| **S2:本机真实研究数据** | 用户自己的论文、笔记、会话、实验和项目状态 | 仅限隔离的本机 UAT | 永不提交、永不上传、永不进入公开截图或 CI | +| **S3:秘密与环境身份** | API key、token、Cookie、邮箱、本机用户名、绝对路径、环境变量、私有远程地址 | 仅在必要的本机私有配置中 | 禁止进入测试输入、报告、日志、artifact、提交和文档示例 | + +公开验收以 S0 和合规的 S1 为主。S2 只能证明真实使用场景可行,不能复制到公开仓库来“展示效果”。S3 不属于产品数据,任何发现都应视为发布阻断问题。 + +## 7. 本机有哪些数据可以做私有 UAT + +本机已知有以下数据类别可供验证,但这里只说明用途,不记录任何真实正文、题名、人员或绝对路径: + +| 本机数据类别 | 适合验证什么 | 建议首轮规模 | +|---|---|---| +| Zotero 本地论文库 | 文献盘点、KB 导出、单篇解析、阅读状态 | 只选 1–2 篇 | +| 论文写作目录中的 PDF、Markdown、Word、表格和演示文稿 | `inventory` 类型识别、Markdown/PDF 安全迁移、交付物关联 | 1 个 Markdown + 1 个 PDF | +| Codex 本地会话日志 | 项目解析、候选总结、审批、`pull -> resume` | 1 个已知项目会话 | +| Claude Code 本地会话日志 | 与 Codex 路径的行为差异和 SessionEnd 接线 | 1 个已知项目会话 | +| 现有 Markdown 研究库 | 项目笔记识别、进展恢复、人工内容保护 | 复制 1 个最小项目到隔离区 | +| 现有 Provenance 收件箱与记忆 | 与旧状态的只读对照、重复导入和恢复检查 | 只读对照,不直接写入 | +| 小型、可重复的本地科研计算项目 | 未来实验账本的运行、指标、失败记录和哈希验证 | 1 个分钟级确定性实验 | +| 大型科研数据库或重型计算数据 | 后续性能与容量测试 | 不用于首轮 UAT | + +上述数据全部属于 S2。**不得上传到 GitHub、CI、第三方模型、网页工具或公开 issue。** 如果实验需要联网模型或云解析器,应改用 S0,或者先逐项确认允许发送的最小材料。 + +## 8. 隔离 UAT 操作步骤 + +1. 新建一个专用 UAT 根目录,内部再分开建立 `workspace`、`provenance-home`、`inputs` 和 `outputs`;不要把测试目录嵌套在当前真实库中。 +2. 保持原始来源只读。首轮只复制一个 Markdown、一个 PDF、一个 Agent 会话和一个小实验所需的最小输入。 +3. 在执行前记录输入文件清单、字节大小和 SHA-256;记录可以保存在本机私有 UAT 报告中,但不要公开绝对路径。 +4. 先用 S0 完成相同命令的 synthetic golden flow,再切换到 S2。 +5. 对所有会写入的命令先运行 preview 或 dry-run,确认目标根目录、数量和动作类型。 +6. 明确把 `workspace`、`provenance-home`、临时目录和配置目录指向 UAT 根目录;不得复用真实 Provenance home、真实 vault 或默认用户配置。 +7. 关闭非必要网络。若某一步必须联网,先确认只发送公开题名、标识符或已经批准的测试材料。 +8. 每一步后检查: + - 原始输入哈希未变化; + - 没有写到 UAT 根目录以外; + - 报告没有正文、邮箱、密钥和绝对路径; + - 待审批内容没有自动变成正式状态。 +9. 重复运行相同步骤,验证没有重复记录、重复复制或字节漂移。 +10. 用 `migrate verify` 检查批次后,在隔离副本上执行 `migrate rollback`;回滚只能删除本批次确认创建且身份、字节均未变的副本;能由记录属性识别的替代文件不会被删除或覆盖,恢复受阻时必须保留 quarantine 并 fail closed。该验收只覆盖单用户下的协作进程,不覆盖恶意本地替换。 +11. 用新会话运行 `resume`,确认只恢复已接受的状态,拒绝项、原始聊天和未审批草稿没有进入胶囊。 +12. 最终 UAT 报告只记录版本、操作系统、通过/失败、计数、匿名 ID、耗时和已知限制。公开前再做一次秘密、路径和私人关键词扫描。 + +## 9. “流程跑通”不等于“科学有效” + +这是整个产品验收中最重要的边界。 + +**工作流成功**可以证明: + +- 文件符合 Schema; +- 会话能恢复; +- 引用和产物有来源; +- 命令可重复; +- 哈希一致; +- 审批边界没有被绕过; +- PPT 能生成并打开。 + +**科学有效**还需要证明: + +- 研究问题可证伪; +- 数据和方法适合该问题; +- 基线比较公平; +- 统计和误差分析成立; +- 结果能复现; +- 反例和冲突证据被认真处理; +- 引用原文确实支持该论断; +- 领域专家或研究者完成最终判断。 + +因此,一条实验记录至少应区分: + +```text +execution_succeeded +result_reproduced +evidence_reviewed +claim_supported +``` + +前一项为真,不能自动推出后一项为真。Context Capsule 中的 `reference_only` 也不能自动升级为 `claim_supported`。 + +## 10. 清理历史垃圾文件时的红线 + +可以列为候选、但必须在测试结束后逐项确认再删除: + +- 本轮明确创建的 synthetic demo 目录; +- 可重新生成的测试缓存、覆盖率文件、构建目录和临时日志; +- 已确认不再需要的隔离 UAT 副本; +- 过期的发布临时克隆或 CI artifact 下载副本。 + +不要因为“体积大”就当成垃圾。虚拟环境和前端依赖通常可重建,但在版本测试完成前应保留;它们与历史隐私垃圾不是同一类问题。 + +绝不能按通配符或整目录清理: + +- 原始私有仓库和保留的私有历史归档; +- Zotero 数据目录及附件; +- 论文、数据集、实验输出和项目笔记; +- Provenance 的 inbox、memory、vault、sync state、profile、搜索索引和输出; +- `.env`、凭据配置和用户级 Agent 配置; +- Codex、Claude Code 或其他 Agent 原始日志; +- 未确认来源的 untracked 文件; +- 任何备份。 + +安全清理流程必须是: + +```text +列出确切绝对目标 + -> 说明为什么可删除、是否可重建、占用大小 + -> 用户逐项确认 + -> 再执行删除 + -> 复查 Git 状态和关键数据仍在 +``` + +在未得到精确目标确认前,只做审计,不做删除。 + +## 11. 最终发布验收的判定方式 + +每个版本都应同时具备四类证据: + +1. **自动化证据**:单元测试、跨仓 E2E、Windows CI、Ubuntu 回归、Schema 验证; +2. **产品证据**:一条从项目建立到下一轮恢复的可观察用户旅程; +3. **隐私证据**:公开仓库、release、artifact、截图和日志的扫描结果; +4. **人工证据**:真实用户完成任务并理解“参考资料、候选结论、已批准状态”三者的区别。 + +只有代码存在,不算完成;只有 demo 跑通,也不算产品完成。V0.4 的真正结束条件是:一个目标科研用户能在干净 Windows 环境独立完成核心闭环,同时没有泄露私有数据,也没有把流程成功误报成科学发现。 diff --git a/docs/case-study.zh-CN.md b/docs/case-study.zh-CN.md index a8de1d9..695d3fa 100644 --- a/docs/case-study.zh-CN.md +++ b/docs/case-study.zh-CN.md @@ -78,8 +78,11 @@ flowchart LR 6. `demo-report.json` 固化版本、阶段、断言、产物和限制,使结果可重复核验。 这里没有 API Key、真实 Zotero、真实对话、在线模型或私人研究资料。Agent 草稿是仓库内 -明确标注的 fixture。Lectern 也没有被算入本条黄金路径,因为生成幻灯片需要独立配置的 -provider 路径。重复运行在功能上幂等,但带摄取时间戳的记录不承诺逐字节一致。 +明确标注的 fixture。主 `scriptorium demo` 仍不调用 Lectern;另有一条离线跨仓验收使用 +`FakeLLM` 和程序生成的合成 PDF,专门验证 `Steward -> handoff/1.1 -> Spec -> +Lectern 生产图 -> 人工提纲审批 -> 可编辑 PPTX`;临时质量检查渲染在进入审批前清理, +不会在输出树中留下未审批 PPTX。重复运行在功能上幂等,但带摄取时间戳的记录 +不承诺逐字节一致。 ## 安全机制如何进入产品细节 @@ -103,7 +106,7 @@ provider 路径。重复运行在功能上幂等,但带摄取时间戳的记 | 远端候选证据链 | commit `8069cbf1` / run `29818934741` / artifact `8490533423` | 通过;旧悬空对象待 GitHub Support 清理 | | 完整 Agent 工作流用户价值 | 尚无外部 beta 行为数据 | 未验证 | | Claude Code SessionEnd 对等路径 | 仍缺 live golden-path 证据 | 未验证 | -| Lectern 跨仓生成 | 不属于当前 credential-free demo | 未验证 | +| Lectern 跨仓生成 | 两篇程序生成的合成 PDF,经 Steward、Spec 和 Lectern 生产图;审批前输出树无持久 PPTX,审批后重新打开并修改原生标题和表格 | 本地自动验收通过;真实 provider 与 PowerPoint 人工打开仍待 UAT | “未验证”不是隐藏在脚注里的免责声明,而是下一阶段产品判断的输入。 @@ -127,10 +130,10 @@ provider 路径。重复运行在功能上幂等,但带摄取时间戳的记 - 当前以 GitHub 源码安装为主,还没有 PyPI/winget 或图形安装器; - editable 源码安装可能从已配置的包索引下载 Python 构建依赖;“运行期不申请网络动作” 不是离线安装承诺; -- `inventory` 当前只做不读取正文的安全盘点与路由,实际迁移仍缺受审阅的 - manifest/apply 闭环; +- `inventory` 仍只做安全盘点与路由;Markdown/PDF 的 + `migrate plan/apply/verify/rollback` 已形成开发候选,但尚未经过远端 Windows CI 和正式发布; - 完整 Public Alpha 需要用户已经安装 Codex 或 Claude Code,并主动安装项目级 Skill; -- Claude Code `SessionEnd` live 对等验证、Lectern schema-driven E2E 与外部用户 beta 仍待完成; +- Claude Code `SessionEnd` live 对等验证、Lectern 真实 provider 路径和外部用户 beta 仍待完成; - demo 的“无网络动作”来自实现与环境白名单,尚未由操作系统级网络沙箱观测; - 尚无真实用户效率数据,因此不宣称已经证明科研质量或节省了确定比例的时间。 diff --git a/pyproject.toml b/pyproject.toml index 9da3767..d5dfe26 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,16 +4,17 @@ build-backend = "setuptools.build_meta" [project] name = "scriptorium-suite" -version = "0.1.0" -description = "Thin, agent-native entry point for the Scriptorium research workflow suite" +version = "0.2.0" +description = "Local-first project context and continuity entry point for long-running work" readme = "README.md" requires-python = ">=3.11" license = { text = "Apache-2.0" } authors = [{ name = "Scriptorium Suite" }] -keywords = ["ai4science", "research", "agents", "workflow"] +keywords = ["agents", "project-memory", "context", "workflow", "ai4science"] classifiers = [ "Development Status :: 3 - Alpha", "Environment :: Console", + "Intended Audience :: Developers", "Intended Audience :: Science/Research", "License :: OSI Approved :: Apache Software License", "Operating System :: Microsoft :: Windows", diff --git a/src/scriptorium/__init__.py b/src/scriptorium/__init__.py index a062522..b234f47 100644 --- a/src/scriptorium/__init__.py +++ b/src/scriptorium/__init__.py @@ -1,3 +1,3 @@ """Scriptorium suite entry point.""" -__version__ = "0.1.0" +__version__ = "0.2.0" diff --git a/src/scriptorium/assets/compatibility.toml b/src/scriptorium/assets/compatibility.toml index a87787a..a40f9f7 100644 --- a/src/scriptorium/assets/compatibility.toml +++ b/src/scriptorium/assets/compatibility.toml @@ -1,6 +1,6 @@ manifest_version = 1 [components] -"scriptorium-spec" = "2.2.0" +"scriptorium-spec" = "2.3.0" steward = "0.2.0" -provenance = "0.17.0" +provenance = "0.18.0" diff --git a/src/scriptorium/assets/component-catalog.toml b/src/scriptorium/assets/component-catalog.toml new file mode 100644 index 0000000..a0732a4 --- /dev/null +++ b/src/scriptorium/assets/component-catalog.toml @@ -0,0 +1,58 @@ +catalog_version = 1 +default_profile = "core" + +[profiles] +core = ["scriptorium-spec", "provenance"] +research = ["scriptorium-spec", "provenance", "steward"] +slides = ["scriptorium-spec", "provenance", "lectern"] +full = ["scriptorium-spec", "provenance", "steward", "lectern"] +capture = ["capture"] + +[components."scriptorium-spec"] +display_name = "Scriptorium Spec" +version = "2.3.0" +repository = "https://github.com/scriptorium-suite/scriptorium-spec.git" +revision = "36612e80c49f5a8f346a09ed73f4b1c67cef5135" +stability = "candidate" +delivery = "source" +directory = "scriptorium-spec" + +[components.provenance] +display_name = "Provenance" +version = "0.18.0" +repository = "https://github.com/foxsplendid/Provenance.git" +revision = "1e3f1c022148c33f257d8a27502b50dbe906611d" +stability = "candidate" +delivery = "python-source" +directory = "Provenance" + +[components.steward] +display_name = "Steward" +version = "0.2.0" +repository = "https://github.com/scriptorium-suite/steward.git" +revision = "7aafad1b432207905072c769fe527a3d4d1b365e" +stability = "stable" +delivery = "python-source" +directory = "steward" + +[components.lectern] +display_name = "Lectern" +version = "0.3.0-candidate" +repository = "https://github.com/foxsplendid/Academic-Slides-Agent.git" +revision = "6371f1e6eac41f505aed39e58dafb689317cab65" +stability = "candidate" +delivery = "workspace-source" +directory = "Academic-Slides-Agent" + +[components.capture] +display_name = "Scriptorium Local Conversation Export" +version = "1.0.0" +repository = "https://github.com/foxsplendid/Provenance.git" +revision = "1e3f1c022148c33f257d8a27502b50dbe906611d" +stability = "candidate" +delivery = "release-asset" +directory = "capture" +owner = "provenance" +artifact_name = "scriptorium-local-conversation-export-1.0.0.zip" +artifact_status = "source-build-only" +artifact_sha256 = "1a85ec282f54bc9cff0c53bb595d7fb5ed37c5ce0121cb036b9fc80574998db8" diff --git a/src/scriptorium/assets/lineage-graph.v1.json b/src/scriptorium/assets/lineage-graph.v1.json new file mode 100644 index 0000000..5379e7d --- /dev/null +++ b/src/scriptorium/assets/lineage-graph.v1.json @@ -0,0 +1,42 @@ +{ + "schema_version": "lineage-graph/1.0", + "generated_by": "scriptorium synthetic fixture", + "direction": { + "query": "[SYNTHETIC] Physics-informed active learning for catalyst discovery", + "scope_method": "synthetic own-library citation match", + "created": "2026-07-15T10:15:00Z" + }, + "nodes": [ + { + "citekey": "chen2022PhysicsInformed", + "title": "[SYNTHETIC] Physics-informed learning for materials discovery", + "year": "2022", + "cluster": "synthetic-foundations" + }, + { + "citekey": "liu2024ActiveLearning", + "title": "[SYNTHETIC] Active learning for closed-loop materials experiments", + "year": "2024", + "cluster": "synthetic-closed-loop" + } + ], + "edges": [ + { + "from": "liu2024ActiveLearning", + "to": "chen2022PhysicsInformed", + "relation": "cites", + "evidence": "[SYNTHETIC] Generated reference recorded in parsed-paper.v1.json." + } + ], + "clusters": [ + { + "id": "synthetic-foundations", + "label": "[SYNTHETIC] Physics-informed foundations" + }, + { + "id": "synthetic-closed-loop", + "label": "[SYNTHETIC] Closed-loop experiment selection" + } + ], + "timeline": ["chen2022PhysicsInformed", "liu2024ActiveLearning"] +} diff --git a/src/scriptorium/assets/parsed-paper.v1.json b/src/scriptorium/assets/parsed-paper.v1.json new file mode 100644 index 0000000..3621ffe --- /dev/null +++ b/src/scriptorium/assets/parsed-paper.v1.json @@ -0,0 +1,43 @@ +{ + "schema_version": "parsed-paper/1.0", + "id": "liu2024ActiveLearning", + "created": "2026-07-15T09:00:00Z", + "parser": "scriptorium synthetic fixture", + "source_pdf": "fixtures/synthetic-active-learning.pdf", + "metadata": { + "title": "[SYNTHETIC] Active learning for closed-loop materials experiments", + "authors": ["Synthetic Author"], + "year": "2024", + "doi": "10.5555/demo.0002", + "abstract": "[SYNTHETIC] This generated abstract exists only to test the Scriptorium workflow." + }, + "sections": [ + { + "heading": "Synthetic method", + "level": 1, + "text": "[SYNTHETIC] A deterministic fixture ranks fictional catalyst candidates by generated uncertainty values." + }, + { + "heading": "Synthetic result", + "level": 1, + "text": "[SYNTHETIC] The fixture demonstrates traceable evidence flow and makes no scientific performance claim." + } + ], + "references": [ + { + "raw": "Chen (2022). [SYNTHETIC] Physics-informed learning for materials discovery.", + "title": "[SYNTHETIC] Physics-informed learning for materials discovery", + "authors": ["Chen"], + "year": "2022", + "doi": "10.5555/demo.0001" + } + ], + "figures": [ + { + "id": "fig-synthetic-loop", + "caption": "[SYNTHETIC] Generated closed-loop selection diagram.", + "page": 2 + } + ], + "tables": [] +} diff --git a/src/scriptorium/assets/reading-note.v1.json b/src/scriptorium/assets/reading-note.v1.json new file mode 100644 index 0000000..955f993 --- /dev/null +++ b/src/scriptorium/assets/reading-note.v1.json @@ -0,0 +1,30 @@ +{ + "schema_version": "reading-note/1.0", + "id": "liu2024ActiveLearning", + "zotero_key": "DEMO0002", + "doi": "10.5555/demo.0002", + "read_status": "In Progress", + "created": "2026-07-15T09:30:00Z", + "generated_by": "scriptorium synthetic fixture", + "stages": { + "glance": { + "tldr": "[SYNTHETIC] Generated note about combining calibrated active learning with physics-informed priors.", + "tags": ["synthetic", "active-learning", "calibration"], + "key_findings": [ + "[SYNTHETIC] The fixture records uncertainty calibration as a candidate design concern." + ] + }, + "close_read": { + "question": "[SYNTHETIC] Can a closed loop prioritize informative fictional catalyst experiments?", + "method": "[SYNTHETIC] Deterministic ranking over generated candidate rows.", + "data": "[SYNTHETIC] Twelve generated catalyst candidates with no real-world meaning.", + "results": "[SYNTHETIC] The workflow records a reviewable result; it does not establish scientific validity.", + "figures": ["fig-synthetic-loop"] + } + }, + "sources": { + "annotations": ["synthetic-note#calibration"], + "parsed_file": "fixtures/parsed-paper.v1.json", + "zotero_uri": "zotero://select/library/items/DEMO0002" + } +} diff --git a/src/scriptorium/assets/review.v1.json b/src/scriptorium/assets/review.v1.json new file mode 100644 index 0000000..a88f7ad --- /dev/null +++ b/src/scriptorium/assets/review.v1.json @@ -0,0 +1,31 @@ +{ + "schema_version": "review/1.0", + "direction": { + "query": "[SYNTHETIC] Physics-informed active learning for catalyst discovery", + "created": "2026-07-15T10:00:00Z" + }, + "sections": [ + { + "heading": "[SYNTHETIC] Candidate direction", + "prose": "[SYNTHETIC] The generated comparison suggests testing whether physics-informed priors and calibrated acquisition can be evaluated together. This is a workflow fixture, not a scientific conclusion." + } + ], + "comparison_table": { + "columns": ["Synthetic role", "Review state"], + "rows": [ + { + "citekey": "chen2022PhysicsInformed", + "cells": ["Generated prior", "Read"] + }, + { + "citekey": "liu2024ActiveLearning", + "cells": ["Generated acquisition loop", "In Progress"] + } + ] + }, + "gaps": [ + "[SYNTHETIC] No deterministic experiment yet links calibration quality to candidate-selection efficiency." + ], + "priority_reads": ["liu2024ActiveLearning"], + "lineage_ref": "fixtures/lineage-graph.v1.json" +} diff --git a/src/scriptorium/assets/skills/scriptorium-research/SKILL.md b/src/scriptorium/assets/skills/scriptorium-research/SKILL.md index 572f2ff..5d11c21 100644 --- a/src/scriptorium/assets/skills/scriptorium-research/SKILL.md +++ b/src/scriptorium/assets/skills/scriptorium-research/SKILL.md @@ -1,34 +1,44 @@ --- name: scriptorium-research -description: Advance an AI4Science project from a vague intuition or existing evidence into reviewable plans, literature artifacts, experiments, papers, and slide handoffs in a Scriptorium Markdown workspace. Use when the user asks to start, resume, investigate, synthesize, plan, review, report, sync, or close out research with Scriptorium, Steward, Provenance, or Lectern, including when scriptorium pull reports agent-fill or project-resolution. +description: Continue a long-running project from user-owned local context into reviewable decisions, actions, checkpoints, and deliverables. Supports general, engineering, personal software, and research profiles in a Scriptorium Markdown workspace. The skill id is retained for compatibility with earlier research-first releases. --- -# Scriptorium Research +# Scriptorium Project Continuity -Use Scriptorium as the orchestration layer for a research workflow. Keep Markdown and +Use Scriptorium as the continuity layer for a long-running project. Keep Markdown and versioned contract files as the collaboration surface; treat each component's public -CLI, read-only MCP tools, and owned files as its stable interface. +CLI, read-only MCP tools, and owned files as its stable interface. The installed skill +directory keeps the legacy `scriptorium-research` name so existing workspaces continue +to resolve it; research is one supported profile rather than the product boundary. -## Advance the research in a verifiable loop +## Advance the project in a verifiable loop 1. Orient to the workspace. - - Identify the workspace, project, research question, current stage, and requested + - Identify the workspace, project objective, selected profile, current stage, and requested deliverable. Run `scriptorium doctor --json --workspace <path>` when available. + - At the start of a continuing project, prefer + `scriptorium resume --project <project-id> --json` or Provenance MCP + `get_context_capsule`. Use the bounded capsule as navigation; retrieve older + sessions or source text only when the current task needs them. - Read the relevant `Projects/*.md` note and existing contract artifacts before proposing changes. - - Query Provenance through `get_current_context`, `get_portfolio`, or - `search_brain` when available. Treat memory as retrieval context, never as a - replacement for source files or primary evidence. + - Fall back to `get_current_context`, `get_portfolio`, or `search_brain` when the + capsule entry is unavailable. Treat memory and `reference_only` capsule items as + retrieval context, never as approved claims or replacements for primary evidence. 2. Frame the next decision. - Restate the question, evidence already available, assumptions, constraints, and a concrete success check. - - Build a compact research frame: falsifiable hypothesis or decision, competing - explanations, an observation that would weaken the preferred explanation, the - smallest discriminating analysis or experiment, and the intended deliverable. + - For a general, engineering, or software project, state the decision, constraints, + alternatives, smallest verifiable change, rollback or failure signal, and intended + deliverable. For a research profile, additionally state the falsifiable hypothesis, + competing explanations, and the smallest discriminating analysis or experiment. - Ask one focused question when a missing answer would materially change the work; otherwise state the assumption and continue. - Choose the smallest useful next step instead of expanding the project silently. -3. Gather and assess evidence. +3. Gather and assess sources. + - Prefer the project's authoritative Markdown, documents, repositories, issue records, + runbooks, logs, and prior reviewed decisions. Treat generated summaries as navigation, + not as replacements for those sources. - For literature work, prefer existing `library-kb`, `parsed-paper`, `reading-note`, `lineage-graph`, and `review` artifacts. Use Steward's public commands and installed literature skills when they fit. diff --git a/src/scriptorium/cli.py b/src/scriptorium/cli.py index 92b0c7a..519c60c 100644 --- a/src/scriptorium/cli.py +++ b/src/scriptorium/cli.py @@ -9,6 +9,11 @@ from pathlib import Path from . import __version__ +from .components import ( + ComponentCatalogError, + build_component_report, + format_component_report, +) from .config import ConfigError, SuiteConfig, load_config from .demo import DemoError, run_demo from .doctor import DoctorError, TARGETS, format_doctor_report, run_doctor @@ -19,7 +24,20 @@ run_host_install, ) from .inventory import format_inventory_report, run_inventory -from .init import InitError, format_init_report, run_init +from .init import PROJECT_TEMPLATES, InitError, format_init_report, run_init +from .installer import InstallError, execute_install, format_install_report, plan_install +from .migration import ( + MIGRATION_LIMITATIONS, + REPORT_VERSION as MIGRATION_REPORT_VERSION, + MigrationError, + apply_migration, + format_migration_report, + load_migration, + plan_migration, + reapply_migration, + rollback_migration, + verify_migration, +) from .path_selection import ( attach_path_selection, codex_home_selection, @@ -28,6 +46,7 @@ selection_warnings, ) from .pull import PullError, format_pull_report, run_pull +from .resume import ResumeError, format_resume_report, run_resume from .status import StatusError, format_status_report, run_status @@ -55,11 +74,49 @@ def build_parser(*, json_errors: bool = False) -> argparse.ArgumentParser: parser_class = _JsonArgumentParser if json_errors else argparse.ArgumentParser parser = parser_class( prog="scriptorium", - description="Agent-native research workflow suite (Public Alpha candidate).", + description="Local-first project context and continuity suite.", ) parser.add_argument("--version", action="version", version=f"scriptorium {__version__}") commands = parser.add_subparsers(dest="command", required=True) + components = commands.add_parser( + "components", + help="show the pinned suite components and selectable product profiles", + ) + components.add_argument( + "--profile", + default="core", + help="component profile to inspect (default: core)", + ) + components.add_argument( + "--json", action="store_true", dest="json_output", help="write JSON to stdout" + ) + + install = commands.add_parser( + "install", + help="preview or install one pinned component profile", + ) + install.add_argument("profile") + install.add_argument( + "--target", + type=Path, + required=True, + help="empty or Scriptorium-owned component directory", + ) + install.add_argument( + "--asset", + type=Path, + help="downloaded release asset for an artifact-only profile", + ) + install.add_argument( + "--run", + action="store_true", + help="clone and prepare the exact reviewed component set", + ) + install.add_argument( + "--json", action="store_true", dest="json_output", help="write JSON to stdout" + ) + demo = commands.add_parser( "demo", help="run the offline, credential-free synthetic research walkthrough", @@ -76,12 +133,18 @@ def build_parser(*, json_errors: bool = False) -> argparse.ArgumentParser: initialize = commands.add_parser( "init", - help="preview or create a minimal real research workspace and suite config", + help="preview or create a minimal project workspace and suite config", ) initialize.add_argument("--workspace", type=Path, required=True) initialize.add_argument("--provenance-home", type=Path, required=True) initialize.add_argument("--project-id", required=True) initialize.add_argument("--title", required=True) + initialize.add_argument( + "--template", + choices=PROJECT_TEMPLATES, + default="general", + help="project note template (default: general)", + ) initialize.add_argument( "--host", action="append", @@ -96,8 +159,10 @@ def build_parser(*, json_errors: bool = False) -> argparse.ArgumentParser: help="existing session working directory (default: workspace)", ) initialize.add_argument( + "--context", "--idea", - help="optional user-authored research intuition for the project note", + dest="idea", + help="optional user-authored initial context for the project note", ) initialize.add_argument( "--config-dir", @@ -147,7 +212,7 @@ def build_parser(*, json_errors: bool = False) -> argparse.ArgumentParser: pull.add_argument( "--workspace", type=Path, - help="existing Markdown research workspace (or use suite config)", + help="existing Scriptorium project workspace (or use suite config)", ) pull.add_argument( "--provenance-home", @@ -173,9 +238,32 @@ def build_parser(*, json_errors: bool = False) -> argparse.ArgumentParser: help="configuration family root used when workspace/data paths are omitted", ) + resume = commands.add_parser( + "resume", + help="read a bounded, reviewable project context capsule", + ) + resume.add_argument( + "--provenance-home", + type=Path, + help="existing Provenance data root (or use suite config)", + ) + resume.add_argument("--provenance-root", type=Path, help="source checkout of Provenance") + resume.add_argument( + "--project", + help="registered project id (or use the suite default project)", + ) + resume.add_argument( + "--json", action="store_true", dest="json_output", help="write JSON to stdout" + ) + resume.add_argument( + "--config-dir", + type=Path, + help="configuration family root used when data root/project are omitted", + ) + status = commands.add_parser( "status", - help="show content-free readiness and pending research workflow counts", + help="show content-free readiness and pending project workflow counts", ) status.add_argument( "--json", action="store_true", dest="json_output", help="write JSON to stdout" @@ -187,7 +275,7 @@ def build_parser(*, json_errors: bool = False) -> argparse.ArgumentParser: status.add_argument( "--workspace", type=Path, - help="existing Markdown research workspace (or use suite config)", + help="existing Scriptorium project workspace (or use suite config)", ) status.add_argument( "--provenance-home", @@ -206,7 +294,7 @@ def build_parser(*, json_errors: bool = False) -> argparse.ArgumentParser: inventory = commands.add_parser( "inventory", - help="inventory explicit local research sources and preview safe routing", + help="inventory explicit local project sources and preview safe routing", ) inventory.add_argument( "--source", @@ -233,6 +321,71 @@ def build_parser(*, json_errors: bool = False) -> argparse.ArgumentParser: "--json", action="store_true", dest="json_output", help="write JSON to stdout" ) + migrate = commands.add_parser( + "migrate", + help="plan, apply, verify, or roll back an explicit Markdown/PDF migration", + ) + migrate_commands = migrate.add_subparsers( + dest="migrate_command", required=True + ) + + def add_migration_identity(command: argparse.ArgumentParser) -> None: + command.add_argument( + "--workspace", + type=Path, + required=True, + help="existing Scriptorium project workspace", + ) + command.add_argument( + "--batch-id", + required=True, + help="stable local migration batch identifier", + ) + command.add_argument( + "--json", + action="store_true", + dest="json_output", + help="write aggregate JSON to stdout", + ) + + migrate_plan = migrate_commands.add_parser( + "plan", + help="preview an explicit migration without writing", + ) + add_migration_identity(migrate_plan) + migrate_plan.add_argument( + "--source", + action="append", + type=Path, + required=True, + help="selected Markdown/PDF file or directory; repeat as needed", + ) + + migrate_apply = migrate_commands.add_parser( + "apply", + help="apply selected sources or resume an existing batch", + ) + add_migration_identity(migrate_apply) + migrate_apply.add_argument( + "--source", + action="append", + type=Path, + default=[], + help="required for a new batch; omit to resume an existing batch", + ) + + migrate_verify = migrate_commands.add_parser( + "verify", + help="verify an existing batch by workspace and batch identifier", + ) + add_migration_identity(migrate_verify) + + migrate_rollback = migrate_commands.add_parser( + "rollback", + help="remove unchanged files owned by an existing batch", + ) + add_migration_identity(migrate_rollback) + host = commands.add_parser( "host", help="manage explicit, project-scoped agent host adapters", @@ -240,13 +393,13 @@ def build_parser(*, json_errors: bool = False) -> argparse.ArgumentParser: host_commands = host.add_subparsers(dest="host_command", required=True) host_install = host_commands.add_parser( "install", - help="install the canonical research skill without changing global host settings", + help="install the canonical project skill without changing global host settings", ) host_install.add_argument("host", choices=HOSTS, help="agent host to configure") host_install.add_argument( "--workspace", type=Path, - help="existing research workspace (or use suite config)", + help="existing Scriptorium project workspace (or use suite config)", ) host_install.add_argument( "--dry-run", @@ -376,6 +529,30 @@ def _pull_error_report(*, run: bool) -> dict[str, object]: } +def _resume_error_report() -> dict[str, object]: + return { + "format_version": 1, + "generated_by": {"name": "scriptorium", "version": __version__}, + "operation": "resume", + "status": "error", + "exit_code": 2, + "capsule": None, + "egress": { + "suite_managed": "not-requested", + "host_managed": "not-invoked", + "optional_connectors": "not-invoked", + }, + "entry": { + "public_command": "prov-context", + "component_exit_code": None, + "stdout": "suppressed", + "stderr": "suppressed", + }, + "errors": [{"code": "entry_error"}], + "limitations": ["No trusted context capsule was available."], + } + + def _status_error_report() -> dict[str, object]: return { "format_version": 1, @@ -445,15 +622,56 @@ def _inventory_error_report() -> dict[str, object]: } +def _migration_error_report( + *, operation: str, code: str = "entry_error" +) -> dict[str, object]: + if operation not in {"plan", "apply", "verify", "rollback"}: + operation = "migration" + return { + "schema_version": MIGRATION_REPORT_VERSION, + "operation": operation, + "status": "error", + "summary": { + "sources_requested": 0, + "files": 0, + "markdown": 0, + "pdf": 0, + "bytes": 0, + "changed": 0, + "unchanged": 0, + }, + "errors": [{"code": code}], + "limitations": list(MIGRATION_LIMITATIONS), + } + + def main(argv: list[str] | None = None) -> int: _configure_output() raw_argv = list(sys.argv[1:] if argv is None else argv) json_command = bool( raw_argv - and raw_argv[0] in {"init", "pull", "status", "inventory"} + and raw_argv[0] in { + "init", + "pull", + "resume", + "status", + "inventory", + "migrate", + "components", + "install", + } and "--json" in raw_argv ) - private_usage_command = bool(raw_argv and raw_argv[0] == "inventory") + private_usage_command = bool( + raw_argv and raw_argv[0] in {"inventory", "resume", "migrate"} + ) + migration_operation = ( + raw_argv[1] + if len(raw_argv) > 1 + and raw_argv[0] == "migrate" + and raw_argv[1] in {"plan", "apply", "verify", "rollback"} + else "migration" + ) try: args = build_parser( json_errors=json_command or private_usage_command @@ -461,7 +679,8 @@ def main(argv: list[str] | None = None) -> int: except _JsonUsageError: if private_usage_command and not json_command: print( - "ERROR: invalid inventory invocation; review scriptorium inventory --help.", + f"ERROR: invalid {raw_argv[0]} invocation; " + f"review scriptorium {raw_argv[0]} --help.", file=sys.stderr, ) return 2 @@ -469,8 +688,14 @@ def main(argv: list[str] | None = None) -> int: error_report = _init_error_report(run="--run" in raw_argv) elif raw_argv and raw_argv[0] == "pull": error_report = _pull_error_report(run="--run" in raw_argv) + elif raw_argv and raw_argv[0] == "resume": + error_report = _resume_error_report() elif raw_argv and raw_argv[0] == "inventory": error_report = _inventory_error_report() + elif raw_argv and raw_argv[0] == "migrate": + error_report = _migration_error_report( + operation=migration_operation + ) else: error_report = _status_error_report() print( @@ -494,6 +719,63 @@ def main(argv: list[str] | None = None) -> int: print(f"Search evidence: {root / 'workspace' / 'Reports' / 'provenance-search.txt'}") print(f"Report: {demo_report}") return 0 + if args.command == "components": + try: + component_report = build_component_report(args.profile) + except ComponentCatalogError as exc: + if args.json_output: + print( + json.dumps( + { + "format_version": 1, + "operation": "components", + "status": "error", + "exit_code": 2, + "errors": [{"code": "catalog_invalid"}], + }, + ensure_ascii=False, + indent=2, + ) + ) + else: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + if args.json_output: + print(json.dumps(component_report, ensure_ascii=False, indent=2)) + else: + print(format_component_report(component_report)) + return 0 + if args.command == "install": + try: + install_report = ( + execute_install(profile=args.profile, target=args.target, asset=args.asset) + if args.run + else plan_install(profile=args.profile, target=args.target, asset=args.asset) + ) + except InstallError as exc: + if args.json_output: + print( + json.dumps( + { + "format_version": 1, + "operation": "install", + "mode": "run" if args.run else "preview", + "status": "error", + "exit_code": 2, + "errors": [{"code": exc.code}], + }, + ensure_ascii=False, + indent=2, + ) + ) + else: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + if args.json_output: + print(json.dumps(install_report, ensure_ascii=False, indent=2)) + else: + print(format_install_report(install_report)) + return int(install_report["exit_code"]) if args.command == "init": try: init_report = run_init( @@ -504,6 +786,7 @@ def main(argv: list[str] | None = None) -> int: hosts=args.hosts, linked_repo=args.linked_repo, idea=args.idea, + template=args.template, config_dir=args.config_dir, run=args.run, ) @@ -666,6 +949,59 @@ def main(argv: list[str] | None = None) -> int: else: print(format_pull_report(pull_report)) return int(pull_report["exit_code"]) + if args.command == "resume": + selections = { + "provenance_root": root_selection( + args.provenance_root, "SCRIPTORIUM_PROVENANCE_ROOT" + ), + } + warnings: list[dict[str, object]] = [] + try: + needs_config = ( + args.config_dir is not None + or args.provenance_home is None + or args.project is None + ) + suite_config = _load_suite_config(args.config_dir, needed=needs_config) + provenance_home, data_root_selection = select_configured_path( + args.provenance_home, + ("PROVENANCE_HOME",), + suite_config.provenance_home if suite_config else None, + ) + selections = { + "data_root": data_root_selection, + **selections, + } + warnings = selection_warnings(selections) + project = args.project or ( + suite_config.default_project if suite_config is not None else None + ) + if provenance_home is None or project is None: + raise ResumeError( + "Provenance home and project are required via flags, environment, or suite config" + ) + resume_report = run_resume( + provenance_home=provenance_home, + provenance_root=args.provenance_root, + project=project, + ) + attach_path_selection(resume_report, selections, warnings) + except (ConfigError, ResumeError): + if args.json_output: + error_report = _resume_error_report() + attach_path_selection(error_report, selections, warnings) + print(json.dumps(error_report, ensure_ascii=False, indent=2)) + else: + print( + "ERROR: context capsule unavailable; run scriptorium doctor for local diagnostics.", + file=sys.stderr, + ) + return 2 + if args.json_output: + print(json.dumps(resume_report, ensure_ascii=False, indent=2)) + else: + print(format_resume_report(resume_report)) + return int(resume_report["exit_code"]) if args.command == "status": selections = { "spec_root": root_selection(args.spec_root, "SCRIPTORIUM_SPEC_ROOT"), @@ -779,6 +1115,88 @@ def main(argv: list[str] | None = None) -> int: return 2 print(inventory_output) return inventory_exit_code + if args.command == "migrate": + try: + if args.migrate_command == "plan": + migration_result = plan_migration( + args.source, + workspace=args.workspace, + batch_id=args.batch_id, + ) + elif args.migrate_command == "apply": + migration_result = ( + apply_migration( + plan_migration( + args.source, + workspace=args.workspace, + batch_id=args.batch_id, + ) + ) + if args.source + else reapply_migration( + workspace=args.workspace, + batch_id=args.batch_id, + ) + ) + elif args.migrate_command == "verify": + migration_result = verify_migration( + workspace=args.workspace, + batch_id=args.batch_id, + ) + else: + migration_result = rollback_migration( + load_migration( + workspace=args.workspace, + batch_id=args.batch_id, + ) + ) + except MigrationError as exc: + if args.json_output: + print( + json.dumps( + _migration_error_report( + operation=args.migrate_command, + code=exc.code, + ), + ensure_ascii=False, + indent=2, + ) + ) + else: + print( + f"ERROR: migrate {args.migrate_command} failed ({exc.code}).", + file=sys.stderr, + ) + return 2 + # Migration reports are a privacy boundary; unexpected details stay local. + except Exception: + if args.json_output: + print( + json.dumps( + _migration_error_report( + operation=args.migrate_command, + ), + ensure_ascii=False, + indent=2, + ) + ) + else: + print( + f"ERROR: migrate {args.migrate_command} failed (entry_error).", + file=sys.stderr, + ) + return 2 + if args.json_output: + print( + json.dumps( + migration_result.report, + ensure_ascii=False, + indent=2, + ) + ) + else: + print(format_migration_report(migration_result.report)) + return 0 if args.command == "host" and args.host_command == "install": try: needs_config = ( diff --git a/src/scriptorium/components.py b/src/scriptorium/components.py new file mode 100644 index 0000000..517ff0f --- /dev/null +++ b/src/scriptorium/components.py @@ -0,0 +1,230 @@ +"""Versioned component catalog for the Scriptorium product entry point.""" + +from __future__ import annotations + +import re +import tomllib +from dataclasses import dataclass +from importlib import resources + + +ASSET_PACKAGE = "scriptorium.assets" +CATALOG_NAME = "component-catalog.toml" +CATALOG_VERSION = 1 +COMPONENT_ID_RE = re.compile(r"^[a-z][a-z0-9-]*$") +REVISION_RE = re.compile(r"^[0-9a-f]{40}$") +SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +DIRECTORY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +STABILITIES = {"stable", "candidate", "experimental"} +DELIVERIES = {"source", "python-source", "workspace-source", "release-asset"} + + +class ComponentCatalogError(RuntimeError): + """The packaged component catalog is missing or internally inconsistent.""" + + +@dataclass(frozen=True) +class Component: + component_id: str + display_name: str + version: str + repository: str + revision: str + stability: str + delivery: str + directory: str + owner: str | None = None + artifact_name: str | None = None + artifact_status: str | None = None + artifact_sha256: str | None = None + + +@dataclass(frozen=True) +class ComponentCatalog: + default_profile: str + profiles: dict[str, tuple[str, ...]] + components: dict[str, Component] + + +def _required_string(data: dict[str, object], name: str, *, where: str) -> str: + value = data.get(name) + if not isinstance(value, str) or not value.strip(): + raise ComponentCatalogError(f"{where}.{name} must be a non-empty string") + return value + + +def _load_component(component_id: str, raw: object) -> Component: + where = f"components.{component_id}" + if not COMPONENT_ID_RE.fullmatch(component_id) or not isinstance(raw, dict): + raise ComponentCatalogError(f"{where} is invalid") + data = dict(raw) + allowed = { + "display_name", + "version", + "repository", + "revision", + "stability", + "delivery", + "directory", + "owner", + "artifact_name", + "artifact_status", + "artifact_sha256", + } + extra = set(data) - allowed + if extra: + raise ComponentCatalogError(f"{where} has unsupported fields") + revision = _required_string(data, "revision", where=where) + if not REVISION_RE.fullmatch(revision): + raise ComponentCatalogError(f"{where}.revision must be a full lowercase commit id") + repository = _required_string(data, "repository", where=where) + if not repository.startswith("https://github.com/") or not repository.endswith(".git"): + raise ComponentCatalogError(f"{where}.repository must be an HTTPS GitHub Git URL") + stability = _required_string(data, "stability", where=where) + if stability not in STABILITIES: + raise ComponentCatalogError(f"{where}.stability is invalid") + delivery = _required_string(data, "delivery", where=where) + if delivery not in DELIVERIES: + raise ComponentCatalogError(f"{where}.delivery is invalid") + owner = data.get("owner") + artifact_name = data.get("artifact_name") + artifact_status = data.get("artifact_status") + artifact_sha256 = data.get("artifact_sha256") + for name, value in ( + ("owner", owner), + ("artifact_name", artifact_name), + ("artifact_status", artifact_status), + ("artifact_sha256", artifact_sha256), + ): + if value is not None and (not isinstance(value, str) or not value.strip()): + raise ComponentCatalogError(f"{where}.{name} must be a non-empty string") + if delivery == "release-asset" and not artifact_name: + raise ComponentCatalogError(f"{where}.artifact_name is required") + if delivery == "release-asset" and ( + not isinstance(artifact_sha256, str) + or not SHA256_RE.fullmatch(artifact_sha256) + ): + raise ComponentCatalogError(f"{where}.artifact_sha256 is required") + directory = _required_string(data, "directory", where=where) + if not DIRECTORY_RE.fullmatch(directory) or directory in {".", ".."}: + raise ComponentCatalogError(f"{where}.directory must be one safe path segment") + return Component( + component_id=component_id, + display_name=_required_string(data, "display_name", where=where), + version=_required_string(data, "version", where=where), + repository=repository, + revision=revision, + stability=stability, + delivery=delivery, + directory=directory, + owner=owner, + artifact_name=artifact_name, + artifact_status=artifact_status, + artifact_sha256=artifact_sha256, + ) + + +def load_component_catalog() -> ComponentCatalog: + try: + payload = resources.files(ASSET_PACKAGE).joinpath(CATALOG_NAME).read_bytes() + raw = tomllib.loads(payload.decode("utf-8")) + except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError) as exc: + raise ComponentCatalogError("component catalog is unreadable") from exc + if set(raw) != {"catalog_version", "default_profile", "profiles", "components"}: + raise ComponentCatalogError("component catalog top-level fields are invalid") + if raw["catalog_version"] != CATALOG_VERSION: + raise ComponentCatalogError("unsupported component catalog version") + raw_components = raw["components"] + raw_profiles = raw["profiles"] + if not isinstance(raw_components, dict) or not isinstance(raw_profiles, dict): + raise ComponentCatalogError("components and profiles must be tables") + components = { + component_id: _load_component(component_id, value) + for component_id, value in raw_components.items() + } + for component in components.values(): + if component.owner is not None and component.owner not in components: + raise ComponentCatalogError( + f"components.{component.component_id}.owner is unknown" + ) + profiles: dict[str, tuple[str, ...]] = {} + for profile, value in raw_profiles.items(): + if not COMPONENT_ID_RE.fullmatch(profile) or not isinstance(value, list): + raise ComponentCatalogError(f"profiles.{profile} is invalid") + if not value or not all(isinstance(item, str) for item in value): + raise ComponentCatalogError(f"profiles.{profile} must be a non-empty string array") + component_ids = tuple(value) + if len(set(component_ids)) != len(component_ids): + raise ComponentCatalogError(f"profiles.{profile} contains duplicates") + if any(component_id not in components for component_id in component_ids): + raise ComponentCatalogError(f"profiles.{profile} references an unknown component") + profiles[profile] = component_ids + default_profile = raw["default_profile"] + if not isinstance(default_profile, str) or default_profile not in profiles: + raise ComponentCatalogError("default_profile is unknown") + return ComponentCatalog( + default_profile=default_profile, + profiles=profiles, + components=components, + ) + + +def build_component_report(profile: str | None = None) -> dict[str, object]: + catalog = load_component_catalog() + selected_profile = profile or catalog.default_profile + if selected_profile not in catalog.profiles: + raise ComponentCatalogError("unknown component profile") + selected = set(catalog.profiles[selected_profile]) + rows = [] + for component_id, component in catalog.components.items(): + rows.append( + { + "id": component_id, + "display_name": component.display_name, + "version": component.version, + "stability": component.stability, + "delivery": component.delivery, + "directory": component.directory, + "selected": component_id in selected, + "owner": component.owner, + "artifact_name": component.artifact_name, + "artifact_status": component.artifact_status, + "artifact_sha256": component.artifact_sha256, + "repository": component.repository, + "revision": component.revision, + } + ) + return { + "format_version": 1, + "operation": "components", + "status": "ok", + "exit_code": 0, + "default_profile": catalog.default_profile, + "selected_profile": selected_profile, + "profiles": { + name: list(component_ids) + for name, component_ids in catalog.profiles.items() + }, + "components": rows, + } + + +def format_component_report(report: dict[str, object]) -> str: + lines = [ + "Scriptorium component catalog", + f"Selected profile: {report['selected_profile']}", + "", + ] + for row in report["components"]: + marker = "*" if row["selected"] else " " + ownership = f"; owned by {row['owner']}" if row["owner"] else "" + availability = ( + f"; {row['artifact_status']}" if row["artifact_status"] else "" + ) + lines.append( + f"{marker} {row['id']} {row['version']} " + f"[{row['stability']}, {row['delivery']}{ownership}{availability}]" + ) + lines.append("") + lines.append("* selected by this profile") + return "\n".join(lines) diff --git a/src/scriptorium/demo.py b/src/scriptorium/demo.py index 73fdffd..c95080b 100644 --- a/src/scriptorium/demo.py +++ b/src/scriptorium/demo.py @@ -356,12 +356,18 @@ def run_demo( "artifacts": [], "expected_artifacts": [ "fixtures/library-kb.v1.1.json", + "fixtures/parsed-paper.v1.json", + "fixtures/reading-note.v1.json", + "fixtures/review.v1.json", + "fixtures/lineage-graph.v1.json", "workspace/Projects/synthetic-catalyst-discovery.md", "workspace/Reviews/ai4science-materials.md", "workspace/Reports/provenance-search.txt", + "workspace/Reports/context-capsule.json", "workspace/Reports/provenance-mcp.jsonl", "provenance/memory/library.json", "provenance/memory/projects.json", + "provenance/memory/research-artifacts.json", "provenance/search-index.db", "demo-report.json", ], @@ -381,13 +387,21 @@ def run_demo( draft_path = fixtures / "review-draft.v1.json" review_path = workspace / "Reviews" / "ai4science-materials.md" kb_path = fixtures / "library-kb.v1.1.json" + research_paths = [ + fixtures / "parsed-paper.v1.json", + fixtures / "reading-note.v1.json", + fixtures / "review.v1.json", + fixtures / "lineage-graph.v1.json", + ] project_path = workspace / "Projects" / "synthetic-catalyst-discovery.md" search_path = workspace / "Reports" / "provenance-search.txt" + context_path = workspace / "Reports" / "context-capsule.json" mcp_path = workspace / "Reports" / "provenance-mcp.jsonl" workspace_readme = workspace / "README.md" prompt_path = input_path.parent / "REVIEW-PROMPT.md" library_memory_path = provenance_home / "memory" / "library.json" project_memory_path = provenance_home / "memory" / "projects.json" + research_memory_path = provenance_home / "memory" / "research-artifacts.json" search_index_path = provenance_home / "search-index.db" config_dir = root / "config" @@ -397,15 +411,18 @@ def run_demo( report_path, kb_path, draft_path, + *research_paths, project_path, workspace_readme, input_path, prompt_path, review_path, search_path, + context_path, mcp_path, library_memory_path, project_memory_path, + research_memory_path, search_index_path, ], ) @@ -446,10 +463,19 @@ def run_demo( steward = find_script(roots["steward"], "steward") prov_ingest_library = find_script(roots["provenance"], "prov-ingest-library") + prov_ingest_research = find_script(roots["provenance"], "prov-ingest-research") prov_ingest_vault = find_script(roots["provenance"], "prov-ingest-vault") prov_search = find_script(roots["provenance"], "prov-search") prov_mcp = find_script(roots["provenance"], "prov-mcp") - provenance_scripts = [prov_ingest_library, prov_ingest_vault, prov_search, prov_mcp] + prov_context = find_script(roots["provenance"], "prov-context") + provenance_scripts = [ + prov_ingest_library, + prov_ingest_research, + prov_ingest_vault, + prov_search, + prov_mcp, + prov_context, + ] _assert( len({path.parent.resolve() for path in provenance_scripts}) == 1, "Provenance commands resolve to different script environments; install one compatible source checkout", @@ -458,6 +484,8 @@ def run_demo( _assert(validator.is_file(), f"spec validator not found: {validator}") _write_asset("library-kb.v1.1.json", kb_path) + for path in research_paths: + _write_asset(path.name, path) _write_asset("review-draft.v1.json", draft_path) _write_asset("synthetic-project.md", project_path) _write_asset("workspace-readme.md", workspace_readme) @@ -486,6 +514,15 @@ def run_demo( ) report["assertions"]["contract_validated"] = True + _run_stage( + "validate literature research contracts", + [sys.executable, str(validator), *[str(path) for path in research_paths]], + cwd=root, + env=env, + stages=stages, + ) + report["assertions"]["research_contracts_validated"] = True + version_result = _run_stage( "check Steward version", [str(steward), "--version"], @@ -589,6 +626,67 @@ def run_demo( ) report["assertions"]["memory_ingested"] = True + first_ingest = _run_stage( + "Provenance research artifact ingest", + [str(prov_ingest_research), *[str(path) for path in research_paths]], + cwd=root, + env=env, + stages=stages, + ) + _assert( + "added=4 updated=0 total=4" in first_ingest.stdout, + "Provenance did not ingest exactly four new synthetic research artifacts", + ) + research_memory = _load_json(research_memory_path) + research_artifacts = research_memory.get("artifacts") or [] + _assert( + research_memory.get("schema_version") == "research-artifacts/1.0" + and len(research_artifacts) == 4 + and {item.get("artifact_type") for item in research_artifacts} + == {"parsed-paper", "reading-note", "review", "lineage-graph"}, + "Provenance research memory does not contain the four expected artifact types", + ) + first_memory_bytes = research_memory_path.read_bytes() + second_ingest = _run_stage( + "Provenance research artifact idempotency", + [str(prov_ingest_research), *[str(path) for path in reversed(research_paths)]], + cwd=root, + env=env, + stages=stages, + ) + _assert( + "added=0 updated=0 total=4" in second_ingest.stdout + and research_memory_path.read_bytes() == first_memory_bytes, + "repeating research artifact ingestion was not byte-stable and idempotent", + ) + report["assertions"]["research_artifacts_ingested"] = True + report["assertions"]["research_ingest_idempotent"] = True + + context_result = _run_stage( + "Provenance context capsule", + [str(prov_context), "--project", "synthetic-catalyst-discovery", "--json"], + cwd=root, + env=env, + stages=stages, + ) + try: + context_capsule = json.loads(context_result.stdout) + except json.JSONDecodeError as exc: + raise DemoError(f"Provenance context capsule is not valid JSON: {exc}") from exc + capsule_artifacts = context_capsule.get("research_artifacts") or [] + _assert( + context_capsule.get("project", {}).get("project_id") + == "synthetic-catalyst-discovery" + and {item.get("kind") for item in capsule_artifacts} + == {"parsed-paper", "reading-note", "review", "lineage-graph"} + and all(item.get("trust") == "reference_only" for item in capsule_artifacts) + and context_capsule.get("trust", {}).get("research_artifacts") + == "reference_only_not_approved_claims", + "context capsule did not expose all research artifacts through the reference-only boundary", + ) + _write_managed_bytes(root, context_path, context_result.stdout.encode("utf-8")) + report["assertions"]["context_capsule_reference_only"] = True + _run_stage( "Provenance search build", [str(prov_search), "--build"], @@ -626,6 +724,15 @@ def run_demo( "method": "tools/call", "params": {"name": "search_brain", "arguments": {"query": "calibration", "limit": 5}}, }, + { + "jsonrpc": "2.0", + "id": 4, + "method": "tools/call", + "params": { + "name": "get_context_capsule", + "arguments": {"project_id": "synthetic-catalyst-discovery"}, + }, + }, ] request_text = "\n".join(json.dumps(item, separators=(",", ":")) for item in requests) + "\n" mcp_result = _run_stage( @@ -645,7 +752,7 @@ def run_demo( continue if isinstance(response, dict) and isinstance(response.get("id"), int): responses[response["id"]] = response - _assert(set(responses) == {0, 1, 2, 3}, "Provenance MCP did not return all four responses") + _assert(set(responses) == {0, 1, 2, 3, 4}, "Provenance MCP did not return all five responses") initialize = responses[0] _assert( initialize.get("jsonrpc") == "2.0" @@ -664,6 +771,7 @@ def run_demo( portfolio_text = _mcp_text(responses[1], 1) context_text = _mcp_text(responses[2], 2) search_text = _mcp_text(responses[3], 3) + capsule_text = _mcp_text(responses[4], 4) _assert( "Synthetic Catalyst Discovery" in portfolio_text and "Physics-informed learning for materials discovery" in context_text @@ -671,6 +779,15 @@ def run_demo( and "id=lit:DEMO0002" in search_text, "Provenance MCP responses did not surface the expected project and literature context", ) + _assert( + "Context Capsule: Synthetic Catalyst Discovery" in capsule_text + and "Research artifact hints (reference only)" in capsule_text + and "[parsed-paper]" in capsule_text + and "[reading-note]" in capsule_text + and "[review]" in capsule_text + and "[lineage-graph]" in capsule_text, + "Provenance MCP context capsule did not preserve the reference-only research hints", + ) mcp_path.write_bytes(mcp_result.stdout.encode("utf-8")) report["assertions"]["mcp_context_verified"] = True diff --git a/src/scriptorium/doctor.py b/src/scriptorium/doctor.py index c51bfd8..e2efe27 100644 --- a/src/scriptorium/doctor.py +++ b/src/scriptorium/doctor.py @@ -30,6 +30,7 @@ PROBE_TIMEOUT_SECONDS = 5 PROVENANCE_BASE_COMMANDS = ( "prov-ingest-library", + "prov-ingest-research", "prov-ingest-vault", "prov-search", "prov-mcp", @@ -421,7 +422,7 @@ def _provenance_check( required_for=TARGETS, passed=passed, summary=( - f"Provenance source/runtime {expected_version} and four public commands matched" + f"Provenance source/runtime {expected_version} and five public commands matched" if passed else "; ".join(errors) ), @@ -434,7 +435,7 @@ def _provenance_check( }, remediation=( "Install the compatible Provenance checkout in one environment " - "and expose all four public commands." + "and expose all five public commands." ), ), root, @@ -876,6 +877,77 @@ def _entry_pull_check( ) +def _entry_context_check( + target: str, + provenance_root: Path | None, + expected_version: str | None = None, +) -> dict[str, Any]: + command = None + command_parent = None + baseline_commands: dict[str, Path] = {} + baseline_parent = None + runtime_version = None + error = None + try: + if provenance_root is None: + raise DoctorError("compatible Provenance source checkout is unavailable") + command = find_script(provenance_root, "prov-context") + baseline_commands = { + name: find_script(provenance_root, name) + for name in PROVENANCE_BASE_COMMANDS + } + baseline_parents = {path.parent.resolve() for path in baseline_commands.values()} + if len(baseline_parents) != 1: + raise DoctorError( + "baseline Provenance commands resolve from different environments" + ) + baseline_parent = next(iter(baseline_parents)) + command_parent = command.parent.resolve() + if command_parent != baseline_parent: + raise DoctorError( + "public context command resolves from a different Provenance environment" + ) + runtime_version = _run_probe( + [str(command), "--version"], + cwd=provenance_root, + extra_env={"PROVENANCE_HOME": os.devnull}, + ).strip() + if expected_version is not None and runtime_version != expected_version: + raise DoctorError( + f"context runtime version {runtime_version} != {expected_version}" + ) + except (DemoError, DoctorError, OSError, RuntimeError) as exc: + error = str(exc) + passed = error is None + return _check( + "entry.resume", + target=target, + required_for=("public-alpha",), + passed=passed, + summary=( + "Read-only bounded context-capsule entry is available" + if passed + else "The context-capsule resume entry is unavailable or incompatible" + ), + details={ + "command": str(command) if command else None, + "command_parent": str(command_parent) if command_parent else None, + "baseline_commands": { + name: str(path) for name, path in baseline_commands.items() + }, + "baseline_command_parent": str(baseline_parent) if baseline_parent else None, + "implemented": passed, + "expected_version": expected_version, + "runtime_version": runtime_version, + "error": error, + }, + remediation=( + "Install the compatible Provenance build exposing `prov-context`, then " + "reinstall the Scriptorium suite entry in the same environment." + ), + ) + + def _registry_application(executable: str) -> Path | None: if os.name != "nt": return None @@ -1295,6 +1367,9 @@ def run_doctor( host_adapter, _agent_capture_check(target, host_adapter), _entry_pull_check(target, resolved_provenance, expected["provenance"]), + _entry_context_check( + target, resolved_provenance, expected["provenance"] + ), ] ) lectern, _ = _lectern_check(target, lectern_root) diff --git a/src/scriptorium/init.py b/src/scriptorium/init.py index 4fc66d8..764059c 100644 --- a/src/scriptorium/init.py +++ b/src/scriptorium/init.py @@ -1,4 +1,4 @@ -"""Safe, no-clobber initialization for a Scriptorium research workspace.""" +"""Safe, no-clobber initialization for a Scriptorium project workspace.""" from __future__ import annotations @@ -29,6 +29,7 @@ PROJECT_ID_RE = re.compile(r"[a-z0-9]+(?:-[a-z0-9]+)*") PROJECT_SCHEMA_RE = re.compile(r"project/1\.[0-9]+") PROJECT_STATUSES = {"planned", "active", "paused", "done", "archived"} +PROJECT_TEMPLATES = ("general", "research", "engineering", "software") _YAML_IMPLICIT = { "null", "~", "true", "false", "yes", "no", "on", "off", ".nan", ".inf", "+.inf", "-.inf", @@ -92,6 +93,14 @@ def _resolve_managed_root(path: Path, *, label: str) -> Path: _check_existing_components(requested, label=label) if requested.exists() and not requested.is_dir(): raise InitError(f"{label} is not a directory") + if requested == Path(requested.anchor): + raise InitError(f"{label} cannot be a filesystem root") + try: + user_home = Path.home().resolve(strict=False) + except (OSError, RuntimeError): + user_home = None + if user_home is not None and requested.resolve(strict=False) == user_home: + raise InitError(f"{label} cannot be the user home directory") return requested @@ -142,7 +151,9 @@ def _canonical_hosts(hosts: list[str]) -> tuple[str, ...]: return tuple(sorted(set(hosts))) -def _validate_text(project_id: str, title: str, idea: str | None) -> None: +def _validate_text( + project_id: str, title: str, idea: str | None, template: str +) -> None: if not isinstance(project_id, str) or not PROJECT_ID_RE.fullmatch(project_id): raise InitError("project id must be a non-empty kebab-case identifier") if ( @@ -155,12 +166,14 @@ def _validate_text(project_id: str, title: str, idea: str | None) -> None: raise InitError("project title must be non-empty and single-line") if PROGRESS_BEGIN in title or PROGRESS_END in title: raise InitError("project title cannot contain managed progress-log markers") + if template not in PROJECT_TEMPLATES: + raise InitError("project template is not supported") if idea is not None and not isinstance(idea, str): - raise InitError("research idea must be text") + raise InitError("initial project context must be text") if idea is not None and "\x00" in idea: - raise InitError("research idea cannot contain NUL bytes") + raise InitError("initial project context cannot contain NUL bytes") if idea is not None and (PROGRESS_BEGIN in idea or PROGRESS_END in idea): - raise InitError("research idea cannot contain managed progress-log markers") + raise InitError("initial project context cannot contain managed progress-log markers") def _yaml_string(value: str) -> str: @@ -173,19 +186,49 @@ def _project_payload( title: str, linked_repo: Path | None, idea: str | None, + template: str, updated: date, ) -> bytes: repo = str(linked_repo).replace("\\", "/") normalized_idea = idea.replace("\r\n", "\n").replace("\r", "\n") if idea else "" - intuition = ( + initial_context = ( normalized_idea.strip() if normalized_idea.strip() - else "<!-- Capture the initial research intuition here. -->" + else "<!-- Capture the initial project context here. -->" + ) + sections = { + "general": ( + ("Overview", initial_context), + ("Goal and scope", "<!-- Define the desired outcome and boundaries. -->"), + ("Source material", "<!-- Link the authoritative files and references. -->"), + ("Decisions", "<!-- Record reviewed decisions and their rationale. -->"), + ), + "research": ( + ("Research intuition", initial_context), + ("Research question", "<!-- Define the research question after review. -->"), + ("Evidence and literature", "<!-- Add source-backed evidence and literature here. -->"), + ), + "engineering": ( + ("System context", initial_context), + ("Current objective or issue", "<!-- Describe the bounded engineering objective. -->"), + ("Constraints and risks", "<!-- Record operational constraints and known risks. -->"), + ("Decisions", "<!-- Record reviewed architecture and maintenance decisions. -->"), + ), + "software": ( + ("Product intent", initial_context), + ("Requirements", "<!-- Record reviewed user needs and acceptance conditions. -->"), + ("Design decisions", "<!-- Record important product and technical trade-offs. -->"), + ("Current implementation", "<!-- Summarize the current working state. -->"), + ), + } + section_text = "\n\n".join( + f"## {heading}\n\n{body}" for heading, body in sections[template] ) text = f"""--- -schema_version: project/1.0 +schema_version: project/1.1 project_id: {_yaml_string(project_id)} title: {_yaml_string(title)} +profile: {_yaml_string(template)} status: planned stage: "" next_actions: [] @@ -198,17 +241,7 @@ def _project_payload( # {title} -## Research intuition - -{intuition} - -## Research question - -<!-- Define the research question after review. --> - -## Evidence and literature - -<!-- Add source-backed evidence and literature here. --> +{section_text} ## Next actions @@ -1453,12 +1486,13 @@ def run_init( hosts: list[str], linked_repo: Path | None = None, idea: str | None = None, + template: str = "research", config_dir: Path | None = None, run: bool = False, today: date | None = None, ) -> dict[str, Any]: """Plan or create the minimal local-first Scriptorium project structure.""" - _validate_text(project_id, title, idea) + _validate_text(project_id, title, idea, template) if today is not None and not isinstance(today, date): raise InitError("today must be a date") canonical_hosts = _canonical_hosts(hosts) @@ -1497,6 +1531,7 @@ def run_init( title=title, linked_repo=resolved_repo or resolved_workspace, idea=idea, + template=template, updated=today or date.today(), ) try: diff --git a/src/scriptorium/installer.py b/src/scriptorium/installer.py new file mode 100644 index 0000000..f6472a8 --- /dev/null +++ b/src/scriptorium/installer.py @@ -0,0 +1,566 @@ +"""Preview-first installer for pinned Scriptorium component profiles.""" + +from __future__ import annotations + +import json +import hashlib +import os +import shutil +import stat +import subprocess +import zipfile +from pathlib import Path +from pathlib import PurePosixPath +from uuid import uuid4 + +from . import __version__ +from .components import ( + Component, + ComponentCatalog, + ComponentCatalogError, + load_component_catalog, +) + + +MARKER_NAME = ".scriptorium-install.json" +POWERSHELL_ENV_NAME = "scriptorium-env.ps1" +BASH_ENV_NAME = "scriptorium-env.sh" +REPORT_VERSION = 1 +MAX_ASSET_ENTRIES = 100 +MAX_ASSET_BYTES = 10 * 1024 * 1024 +COMMAND_TIMEOUT_SECONDS = 300 + + +class InstallError(RuntimeError): + """A component install plan cannot be executed safely.""" + + def __init__(self, message: str, *, code: str = "install_error") -> None: + super().__init__(message) + self.code = code + + +def _absolute(path: Path) -> Path: + return Path(os.path.abspath(path.expanduser())) + + +def _is_linklike(path: Path) -> bool: + try: + metadata = path.lstat() + except FileNotFoundError: + return False + except OSError as exc: + raise InstallError("cannot inspect install target", code="target_unreadable") from exc + reparse_flag = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x00000400) + return stat.S_ISLNK(metadata.st_mode) or bool( + getattr(metadata, "st_file_attributes", 0) & reparse_flag + ) + + +def _check_target(target: Path) -> None: + current = Path(target.anchor) if target.anchor else Path() + parts = target.parts[1:] if target.anchor else target.parts + for index, part in enumerate(parts): + current /= part + if _is_linklike(current): + raise InstallError( + "install target contains a symlink, junction, or reparse point", + code="target_link", + ) + if current.exists() and index < len(parts) - 1 and not current.is_dir(): + raise InstallError("install target has a non-directory component", code="target_invalid") + if target.exists() and not target.is_dir(): + raise InstallError("install target is not a directory", code="target_invalid") + if target == Path(target.anchor): + raise InstallError("install target cannot be a filesystem root", code="target_too_broad") + try: + user_home = Path.home().resolve(strict=False) + except (OSError, RuntimeError): + user_home = None + if user_home is not None and target.resolve(strict=False) == user_home: + raise InstallError("install target cannot be the user home directory", code="target_too_broad") + + +def _selected_components( + profile: str, catalog: ComponentCatalog +) -> tuple[Component, ...]: + component_ids = catalog.profiles.get(profile) + if component_ids is None: + raise InstallError("unknown component profile", code="unknown_profile") + return tuple(catalog.components[component_id] for component_id in component_ids) + + +def _catalog_or_install_error() -> ComponentCatalog: + try: + return load_component_catalog() + except ComponentCatalogError as exc: + raise InstallError("component catalog is invalid", code="catalog_invalid") from exc + + +def _existing_marker(target: Path) -> dict[str, object] | None: + marker = target / MARKER_NAME + if not marker.exists(): + return None + if _is_linklike(marker) or not marker.is_file(): + raise InstallError("install marker is not a regular file", code="marker_invalid") + try: + data = json.loads(marker.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise InstallError("install marker is unreadable", code="marker_invalid") from exc + if ( + not isinstance(data, dict) + or set(data) != {"format_version", "profile", "state", "components"} + or data.get("format_version") != REPORT_VERSION + or not isinstance(data.get("profile"), str) + or not data.get("profile") + or data.get("state") not in {"installing", "installed", "failed"} + or not isinstance(data.get("components"), dict) + or not all( + isinstance(name, str) + and bool(name) + and isinstance(revision, str) + and len(revision) == 40 + and all(character in "0123456789abcdef" for character in revision) + for name, revision in data["components"].items() + ) + ): + raise InstallError("install marker format is unsupported", code="marker_invalid") + return data + + +def _component_action( + component: Component, *, existing: bool, local_asset_ready: bool +) -> dict[str, object]: + available = component.delivery != "release-asset" or local_asset_ready or ( + component.artifact_status == "published" + ) + return { + "component": component.component_id, + "version": component.version, + "stability": component.stability, + "delivery": component.delivery, + "destination": component.directory, + "revision": component.revision, + "action": "verify" if existing else "install", + "available": available, + "artifact_name": component.artifact_name, + } + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + try: + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + except OSError as exc: + raise InstallError("release asset is unreadable", code="asset_unreadable") from exc + return digest.hexdigest() + + +def _validate_local_asset(component: Component, asset: Path) -> Path: + requested = _absolute(asset) + if _is_linklike(requested) or not requested.is_file(): + raise InstallError("release asset is not a regular file", code="asset_invalid") + try: + if requested.stat().st_size > MAX_ASSET_BYTES: + raise InstallError("release asset is too large", code="asset_invalid") + except OSError as exc: + raise InstallError("release asset is unreadable", code="asset_unreadable") from exc + if requested.name != component.artifact_name: + raise InstallError("release asset filename does not match the catalog", code="asset_invalid") + if _file_sha256(requested) != component.artifact_sha256: + raise InstallError("release asset checksum does not match the catalog", code="asset_checksum") + return requested + + +def plan_install( + *, + profile: str, + target: Path, + asset: Path | None = None, + catalog: ComponentCatalog | None = None, +) -> dict[str, object]: + selected_catalog = catalog or _catalog_or_install_error() + components = _selected_components(profile, selected_catalog) + release_components = tuple( + component for component in components if component.delivery == "release-asset" + ) + if asset is not None and len(release_components) != 1: + raise InstallError( + "--asset requires a profile with exactly one release asset", + code="asset_not_applicable", + ) + local_asset = ( + _validate_local_asset(release_components[0], asset) + if asset is not None + else None + ) + resolved_target = _absolute(target) + _check_target(resolved_target) + marker = _existing_marker(resolved_target) if resolved_target.exists() else None + if resolved_target.exists(): + unexpected = [ + entry.name + for entry in resolved_target.iterdir() + if entry.name != MARKER_NAME + ] + if unexpected and marker is None: + raise InstallError( + "non-empty install target is not owned by Scriptorium", + code="target_not_owned", + ) + for component in components: + destination = resolved_target / component.directory + if destination.exists() and _is_linklike(destination): + raise InstallError("component destination is a link", code="target_link") + actions = [ + _component_action( + component, + existing=(resolved_target / component.directory).exists(), + local_asset_ready=( + local_asset is not None and component in release_components + ), + ) + for component in components + ] + unavailable = sum(not action["available"] for action in actions) + return { + "format_version": REPORT_VERSION, + "generated_by": {"name": "scriptorium", "version": __version__}, + "operation": "install", + "mode": "preview", + "status": "planned" if not unavailable else "action-required", + "exit_code": 0, + "profile": profile, + "actions": actions, + "summary": { + "components": len(actions), + "available": len(actions) - unavailable, + "unavailable": unavailable, + }, + "network": ( + "not-required" + if local_asset is not None and len(release_components) == len(components) + else "required-on-run" + ), + "writes": "none", + "limitations": ( + ["One or more release assets have not been published yet."] + if unavailable + else [] + ), + } + + +def _command_environment() -> dict[str, str]: + blocked_fragments = ("TOKEN", "SECRET", "PASSWORD", "API_KEY") + environment = { + key: value + for key, value in os.environ.items() + if not any(fragment in key.upper() for fragment in blocked_fragments) + } + environment["GIT_TERMINAL_PROMPT"] = "0" + environment["PYTHONUTF8"] = "1" + return environment + + +def _run_command(arguments: list[str], *, cwd: Path | None = None) -> str: + try: + completed = subprocess.run( + arguments, + cwd=cwd, + env=_command_environment(), + check=False, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=COMMAND_TIMEOUT_SECONDS, + ) + except subprocess.TimeoutExpired as exc: + raise InstallError("component install command timed out", code="command_timeout") from exc + except OSError as exc: + raise InstallError("required install command is unavailable", code="command_missing") from exc + if completed.returncode != 0: + raise InstallError("component install command failed", code="command_failed") + return completed.stdout.strip() + + +def _verify_checkout(path: Path, revision: str) -> None: + actual = _run_command(["git", "-C", str(path), "rev-parse", "HEAD"]) + if actual != revision: + raise InstallError("component revision does not match the catalog", code="revision_mismatch") + if _run_command( + ["git", "-C", str(path), "status", "--porcelain=v1", "--untracked-files=normal"] + ): + raise InstallError("component checkout contains local changes", code="revision_mismatch") + + +def _install_source(component: Component, destination: Path) -> None: + _run_command( + [ + "git", + "clone", + "--filter=blob:none", + "--no-checkout", + component.repository, + str(destination), + ] + ) + _run_command( + ["git", "-C", str(destination), "checkout", "--detach", component.revision] + ) + _verify_checkout(destination, component.revision) + if component.delivery in {"python-source", "workspace-source"}: + _run_command(["uv", "sync", "--locked"], cwd=destination) + + +def _safe_archive_entries(archive: zipfile.ZipFile) -> tuple[zipfile.ZipInfo, ...]: + entries = tuple(archive.infolist()) + names = [entry.filename for entry in entries] + folded_names = [name.casefold() for name in names] + if ( + not entries + or len(entries) > MAX_ASSET_ENTRIES + or len(set(names)) != len(names) + or len(set(folded_names)) != len(folded_names) + ): + raise InstallError("release asset entry set is invalid", code="asset_invalid") + total = 0 + for entry in entries: + name = entry.filename + pure = PurePosixPath(name) + mode = (entry.external_attr >> 16) & 0o170000 + if ( + not name + or "\\" in name + or "\x00" in name + or pure.is_absolute() + or any(part in {"", ".", ".."} for part in pure.parts) + or (pure.parts and ":" in pure.parts[0]) + or mode == stat.S_IFLNK + ): + raise InstallError("release asset contains an unsafe entry", code="asset_invalid") + total += entry.file_size + if total > MAX_ASSET_BYTES: + raise InstallError("release asset is too large", code="asset_invalid") + return entries + + +def _install_release_asset( + component: Component, asset: Path, destination: Path +) -> None: + destination.mkdir() + stored = destination / component.artifact_name + shutil.copyfile(asset, stored) + if _file_sha256(stored) != component.artifact_sha256: + raise InstallError("release asset changed while copying", code="asset_checksum") + unpacked = destination / "unpacked" + unpacked.mkdir() + try: + with zipfile.ZipFile(stored, "r") as archive: + entries = _safe_archive_entries(archive) + for entry in entries: + output = unpacked.joinpath(*PurePosixPath(entry.filename).parts) + if entry.is_dir(): + output.mkdir(parents=True, exist_ok=True) + continue + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(archive.read(entry)) + except (OSError, zipfile.BadZipFile, RuntimeError) as exc: + if isinstance(exc, InstallError): + raise + raise InstallError("release asset could not be unpacked", code="asset_invalid") from exc + + +def _verify_release_asset(component: Component, destination: Path) -> None: + stored = destination / str(component.artifact_name) + if _is_linklike(stored) or not stored.is_file(): + raise InstallError("installed release asset is missing", code="asset_invalid") + if _file_sha256(stored) != component.artifact_sha256: + raise InstallError("installed release asset checksum changed", code="asset_checksum") + unpacked = destination / "unpacked" + if _is_linklike(unpacked) or not unpacked.is_dir(): + raise InstallError("installed release asset is incomplete", code="asset_invalid") + try: + with zipfile.ZipFile(stored, "r") as archive: + for entry in _safe_archive_entries(archive): + if entry.is_dir(): + continue + output = unpacked.joinpath(*PurePosixPath(entry.filename).parts) + if _is_linklike(output) or not output.is_file(): + raise InstallError("installed release asset is incomplete", code="asset_invalid") + if output.read_bytes() != archive.read(entry): + raise InstallError("installed release asset content changed", code="asset_checksum") + except zipfile.BadZipFile as exc: + raise InstallError("installed release asset is invalid", code="asset_invalid") from exc + + +def _write_marker( + target: Path, + profile: str, + components: tuple[Component, ...], + *, + state: str, +) -> None: + marker = target / MARKER_NAME + temporary = target / f".{MARKER_NAME}.{uuid4().hex}.tmp" + payload = { + "format_version": REPORT_VERSION, + "profile": profile, + "state": state, + "components": { + component.component_id: component.revision for component in components + }, + } + try: + with temporary.open("x", encoding="utf-8", newline="\n") as stream: + stream.write(json.dumps(payload, ensure_ascii=False, indent=2) + "\n") + except OSError as exc: + raise InstallError("install marker could not be written", code="marker_invalid") from exc + try: + os.replace(temporary, marker) + except OSError as exc: + try: + temporary.unlink(missing_ok=True) + except OSError: + pass + raise InstallError("install marker could not be published", code="marker_invalid") from exc + + +def _environment_roots(components: tuple[Component, ...]) -> dict[str, str]: + roots = {} + for component in components: + if component.component_id == "scriptorium-spec": + roots["SCRIPTORIUM_SPEC_ROOT"] = component.directory + elif component.component_id == "provenance": + roots["SCRIPTORIUM_PROVENANCE_ROOT"] = component.directory + elif component.component_id == "steward": + roots["SCRIPTORIUM_STEWARD_ROOT"] = component.directory + elif component.component_id == "lectern": + roots["SCRIPTORIUM_LECTERN_ROOT"] = component.directory + return roots + + +def _write_environment_scripts( + target: Path, components: tuple[Component, ...] +) -> None: + roots = _environment_roots(components) + powershell = [ + "$ScriptoriumComponents = Split-Path -Parent $MyInvocation.MyCommand.Path" + ] + bash = [ + 'SCRIPTORIUM_COMPONENTS_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"', + "export SCRIPTORIUM_COMPONENTS_ROOT", + ] + for name, directory in roots.items(): + powershell.append( + f"$env:{name} = Join-Path $ScriptoriumComponents '{directory}'" + ) + bash.extend( + [ + f'{name}="$SCRIPTORIUM_COMPONENTS_ROOT/{directory}"', + f"export {name}", + ] + ) + executable_dirs = [ + component.directory + for component in components + if component.delivery in {"python-source", "workspace-source"} + ] + powershell.append("$ScriptoriumCommandPaths = @()") + for directory in executable_dirs: + powershell.extend( + [ + f"$Candidate = Join-Path $ScriptoriumComponents '{directory}\\.venv\\Scripts'", + "if (Test-Path -LiteralPath $Candidate) { $ScriptoriumCommandPaths += $Candidate }", + ] + ) + bash.append( + f'if [ -d "$SCRIPTORIUM_COMPONENTS_ROOT/{directory}/.venv/bin" ]; then ' + f'PATH="$SCRIPTORIUM_COMPONENTS_ROOT/{directory}/.venv/bin:$PATH"; fi' + ) + powershell.append( + "$env:PATH = ($ScriptoriumCommandPaths + @($env:PATH)) -join [IO.Path]::PathSeparator" + ) + bash.append("export PATH") + powershell_path = target / POWERSHELL_ENV_NAME + bash_path = target / BASH_ENV_NAME + if _is_linklike(powershell_path) or _is_linklike(bash_path): + raise InstallError("environment script is a link", code="target_link") + powershell_path.write_text( + "\n".join(powershell) + "\n", encoding="utf-8" + ) + bash_path.write_text("\n".join(bash) + "\n", encoding="utf-8") + + +def execute_install( + *, + profile: str, + target: Path, + asset: Path | None = None, + catalog: ComponentCatalog | None = None, +) -> dict[str, object]: + selected_catalog = catalog or _catalog_or_install_error() + components = _selected_components(profile, selected_catalog) + preview = plan_install( + profile=profile, target=target, asset=asset, catalog=selected_catalog + ) + if preview["summary"]["unavailable"]: + raise InstallError( + "selected profile contains an unpublished release asset", + code="artifact_unpublished", + ) + resolved_target = _absolute(target) + resolved_target.mkdir(parents=True, exist_ok=True) + _write_marker(resolved_target, profile, components, state="installing") + try: + for component in components: + destination = resolved_target / component.directory + if destination.exists(): + if component.delivery == "release-asset": + _verify_release_asset(component, destination) + else: + _verify_checkout(destination, component.revision) + continue + if component.delivery == "release-asset": + if asset is None: + raise InstallError("release-asset download is unavailable", code="artifact_unpublished") + _install_release_asset(component, _validate_local_asset(component, asset), destination) + else: + _install_source(component, destination) + _write_environment_scripts(resolved_target, components) + _write_marker(resolved_target, profile, components, state="installed") + except Exception: + try: + _write_marker(resolved_target, profile, components, state="failed") + except OSError: + pass + raise + report = dict(preview) + report.update( + { + "mode": "run", + "status": "installed", + "writes": "managed-component-directories-and-marker", + } + ) + return report + + +def format_install_report(report: dict[str, object]) -> str: + lines = [ + f"Scriptorium install ({report['mode']})", + f"Profile: {report['profile']}", + f"Result: {str(report['status']).upper()}", + ] + for action in report["actions"]: + availability = "ready" if action["available"] else "not published" + lines.append( + f"- {action['component']} {action['version']}: " + f"{action['action']} ({availability})" + ) + lines.append(f"Network on run: {report['network']}") + if report["mode"] == "run": + lines.append("Next on Windows: . .\\scriptorium-env.ps1") + return "\n".join(lines) diff --git a/src/scriptorium/inventory.py b/src/scriptorium/inventory.py index 7d39096..a36f2ba 100644 --- a/src/scriptorium/inventory.py +++ b/src/scriptorium/inventory.py @@ -1,4 +1,4 @@ -"""Content-free inventory preview for explicitly selected research sources.""" +"""Content-free inventory preview for explicitly selected project sources.""" from __future__ import annotations diff --git a/src/scriptorium/migration.py b/src/scriptorium/migration.py new file mode 100644 index 0000000..ce45634 --- /dev/null +++ b/src/scriptorium/migration.py @@ -0,0 +1,1595 @@ +"""Explicit, local-only Markdown/PDF migration. + +Private manifests contain absolute paths and live outside the project +workspace. Public reports contain aggregate counts only. + +Threat boundary: this module coordinates cooperative processes for one local +user. It does not defend against a malicious local process changing files +during an operation. +""" + +from __future__ import annotations + +import copy +import ctypes +import errno +import hashlib +import json +import os +import re +import secrets +import stat +import sys +import tempfile +from contextlib import contextmanager +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterable, Mapping + + +MANIFEST_VERSION = "migration-manifest/1.0" +REPORT_VERSION = "migration-report/1.0" +MANIFEST_PRIVACY = "local-private" +MIGRATION_LIMITATIONS = ( + "Only explicitly selected Markdown and PDF files or directories are supported.", + "AI conversations, Zotero libraries, parsing, indexing, and network sources are not supported.", + "The private path manifest stays in the local state root outside the project workspace.", + "Coordination assumes one local user and cooperative Scriptorium processes.", + "Atomic publication requires hard-link support on the local destination filesystem.", + "Sources may be on another local volume because bytes are staged beside each destination.", + ( + "A random internal .scriptorium-*.stage hard-link ownership anchor " + "remains beside each migrated target until rollback." + ), + ( + "A crash before an anchor is recorded may leave an unclaimed random stage; " + "it is never adopted or deleted automatically." + ), + ( + "Automatic rollback requires Windows no-replace rename or Linux renameat2; " + "unsupported platforms fail closed." + ), + ( + "Rollback quarantines and re-verifies owned files; foreign replacements are " + "restored or preserved, and empty directories may remain." + ), +) + +_KINDS = {".md": "markdown", ".markdown": "markdown", ".pdf": "pdf"} +_BATCH_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}\Z") +_ID_RE = re.compile(r"file-[0-9]{6}\Z") +_HASH_RE = re.compile(r"[0-9a-f]{64}\Z") +_TOKEN_RE = re.compile(r"[0-9a-f]{32}\Z") +_REPARSE_POINT = 0x400 +_DRIVE_REMOTE = 4 +_ROLLBACK_PHASES = { + "target", + "target-quarantined", + "anchor", + "anchor-quarantined", +} +_AT_FDCWD = -100 +_RENAME_NOREPLACE = 1 +_RUN_STATES = { + "planned", + "applying", + "applied", + "rolling-back", + "rolled-back", +} +_ENTRY_STATES = { + "planned", + "creating", + "created", + "delete-pending", + "deleted", +} +_RUN_ENTRY_STATES = { + "planned": {"planned"}, + "applying": {"planned", "creating", "created"}, + "applied": {"created"}, + "rolling-back": _ENTRY_STATES, + "rolled-back": {"deleted"}, +} +MIGRATION_ERROR_CODES = frozenset( + { + "applied_manifest_missing", + "atomic_publish_unavailable", + "atomic_quarantine_unavailable", + "batch_conflict", + "cross_volume_publish_unsupported", + "entry_error", + "file_identity_unavailable", + "invalid_batch_id", + "invalid_manifest", + "invalid_path", + "invalid_sources", + "invalid_state_transition", + "link_or_reparse_rejected", + "lock_invalid", + "lock_unavailable", + "manifest_integrity_failed", + "migration_already_rolled_back", + "migration_not_applied", + "migration_not_found", + "no_sources_selected", + "no_supported_files", + "noncanonical_state_root", + "owned_target_changed", + "path_unreadable", + "remote_path_rejected", + "rollback_failed", + "rollback_in_progress", + "rollback_restore_blocked", + "source_destination_overlap", + "source_hash_changed", + "source_missing", + "source_not_regular", + "source_state_overlap", + "source_unreadable", + "staging_cleanup_failed", + "staging_conflict", + "quarantine_conflict", + "state_root_in_source", + "state_root_invalid", + "state_workspace_overlap", + "state_write_failed", + "stored_manifest_invalid", + "stored_manifest_missing", + "target_escape", + "target_exists", + "target_unwritable", + "unsupported_source_type", + "workspace_identity_changed", + "workspace_missing", + } +) + + +class MigrationError(RuntimeError): + """A path-free migration failure.""" + + def __init__(self, code: str) -> None: + self.code = code if code in MIGRATION_ERROR_CODES else "entry_error" + super().__init__(f"migration failed: {self.code}") + + +@dataclass(frozen=True, repr=False) +class MigrationPlan: + manifest: dict[str, Any] = field(repr=False) + report: dict[str, Any] = field(repr=False) + + def __repr__(self) -> str: + return _safe_repr(type(self).__name__, self.report) + + +@dataclass(frozen=True, repr=False) +class MigrationResult: + manifest: dict[str, Any] = field(repr=False) + report: dict[str, Any] = field(repr=False) + + def __repr__(self) -> str: + return _safe_repr(type(self).__name__, self.report) + + +def _safe_repr(name: str, report: Mapping[str, Any]) -> str: + summary = report.get("summary", {}) + files = summary.get("files", 0) if isinstance(summary, Mapping) else 0 + return f"{name}(status={report.get('status')!r}, files={files!r})" + + +def _absolute(value: os.PathLike[str] | str) -> Path: + try: + return Path(os.path.abspath(os.fspath(Path(value).expanduser()))) + except (OSError, TypeError, ValueError) as exc: + raise MigrationError("invalid_path") from exc + + +def _inside(root: Path, path: Path) -> bool: + try: + path.relative_to(root) + except ValueError: + return False + return path != root + + +def _linklike(metadata: os.stat_result) -> bool: + attributes = int(getattr(metadata, "st_file_attributes", 0)) + return stat.S_ISLNK(metadata.st_mode) or bool(attributes & _REPARSE_POINT) + + +def _metadata(path: Path) -> os.stat_result | None: + try: + return path.lstat() + except FileNotFoundError: + return None + except OSError as exc: + raise MigrationError("path_unreadable") from exc + + +def _check_components(path: Path) -> None: + current = path + components: list[Path] = [] + while True: + components.append(current) + if current.parent == current: + break + current = current.parent + for component in reversed(components): + metadata = _metadata(component) + if metadata is not None and _linklike(metadata): + raise MigrationError("link_or_reparse_rejected") + + +def _reject_remote(path: Path) -> None: + if os.fspath(path).startswith(("\\\\", "//")): + raise MigrationError("remote_path_rejected") + if os.name != "nt" or not path.anchor: + return + get_drive_type = ctypes.windll.kernel32.GetDriveTypeW + get_drive_type.argtypes = [ctypes.c_wchar_p] + get_drive_type.restype = ctypes.c_uint + if int(get_drive_type(path.anchor)) == _DRIVE_REMOTE: + raise MigrationError("remote_path_rejected") + + +def _existing_directory(value: os.PathLike[str] | str, code: str) -> Path: + path = _absolute(value) + _reject_remote(path) + _check_components(path) + metadata = _metadata(path) + if metadata is None or not stat.S_ISDIR(metadata.st_mode): + raise MigrationError(code) + try: + return path.resolve(strict=True) + except OSError as exc: + raise MigrationError(code) from exc + + +def _ensure_tree(root: Path, directory: Path) -> None: + if directory != root and not _inside(root, directory): + raise MigrationError("target_escape") + _check_components(directory) + try: + root.mkdir(parents=True, exist_ok=True) + directory.mkdir(parents=True, exist_ok=True) + except OSError as exc: + raise MigrationError("target_unwritable") from exc + _check_components(directory) + for candidate in (root, directory): + metadata = _metadata(candidate) + if metadata is None or not stat.S_ISDIR(metadata.st_mode): + raise MigrationError("target_unwritable") + + +def _default_state_root() -> Path: + if os.name == "nt" and os.environ.get("LOCALAPPDATA"): + return _absolute(Path(os.environ["LOCALAPPDATA"]) / "Scriptorium" / "state") + if os.environ.get("XDG_STATE_HOME"): + return _absolute(Path(os.environ["XDG_STATE_HOME"]) / "scriptorium") + return _absolute(Path.home() / ".local" / "state" / "scriptorium") + + +def _same_path(left: Path, right: Path) -> bool: + try: + left_value = os.fspath(left.resolve(strict=False)) + right_value = os.fspath(right.resolve(strict=False)) + except OSError as exc: + raise MigrationError("state_root_invalid") from exc + return os.path.normcase(left_value) == os.path.normcase(right_value) + + +def _private_state_root( + value: os.PathLike[str] | str | None, workspace: Path +) -> Path: + canonical = _default_state_root() + requested = canonical if value is None else _absolute(value) + if not _same_path(requested, canonical): + raise MigrationError("noncanonical_state_root") + root = canonical.resolve(strict=False) + _reject_remote(root) + _check_components(root) + metadata = _metadata(root) + if metadata is not None and not stat.S_ISDIR(metadata.st_mode): + raise MigrationError("state_root_invalid") + if root == workspace or _inside(workspace, root) or _inside(root, workspace): + raise MigrationError("state_workspace_overlap") + return root + + +def _kind(path: Path) -> str | None: + return _KINDS.get(path.suffix.lower()) + + +def _reject_state_source_overlap(source: Path, private_root: Path) -> None: + if source == private_root or _inside(source, private_root): + raise MigrationError("state_root_in_source") + if _inside(private_root, source): + raise MigrationError("source_state_overlap") + + +def _hash_file(path: Path, code: str) -> tuple[str, int]: + metadata = _metadata(path) + if metadata is None: + raise MigrationError(code) + if _linklike(metadata) or not stat.S_ISREG(metadata.st_mode): + raise MigrationError(code) + digest = hashlib.sha256() + size = 0 + try: + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + size += len(chunk) + except OSError as exc: + raise MigrationError(code) from exc + return digest.hexdigest(), size + + +def _selected( + sources: Iterable[os.PathLike[str] | str] | os.PathLike[str] | str, +) -> list[Path]: + if isinstance(sources, (str, os.PathLike)): + values = [sources] + else: + try: + values = list(sources) + except TypeError as exc: + raise MigrationError("invalid_sources") from exc + if not values: + raise MigrationError("no_sources_selected") + if any( + isinstance(value, bytes) or not isinstance(value, (str, os.PathLike)) + for value in values + ): + raise MigrationError("invalid_sources") + return [_absolute(value) for value in values] + + +def _scan(root: Path) -> list[Path]: + found: list[Path] = [] + + def fail(error: OSError) -> None: + raise error + + try: + walker = os.walk(root, topdown=True, onerror=fail, followlinks=False) + for directory, names, files in walker: + names.sort(key=str.casefold) + files.sort(key=str.casefold) + for name in names: + metadata = _metadata(Path(directory) / name) + if metadata is not None and _linklike(metadata): + raise MigrationError("link_or_reparse_rejected") + for name in files: + path = Path(directory) / name + metadata = _metadata(path) + if metadata is not None and _linklike(metadata): + raise MigrationError("link_or_reparse_rejected") + if metadata is not None and stat.S_ISREG(metadata.st_mode) and _kind(path): + found.append(path) + except OSError as exc: + raise MigrationError("source_unreadable") from exc + return sorted(found, key=lambda path: os.path.normcase(os.fspath(path))) + + +def _label(index: int, path: Path, *, file: bool) -> str: + suffix = path.suffix.lower() if file else "" + raw = path.stem if file else path.name + slug = re.sub(r"[^A-Za-z0-9._-]+", "-", raw).strip(".-") or "source" + return f"{index:03d}-{slug[:64]}{suffix}" + + +def _relative(value: str | Path) -> str: + path = Path(value) + if path.is_absolute() or not path.parts: + raise MigrationError("target_escape") + if any(part in {"", ".", ".."} or ":" in part or "\x00" in part for part in path.parts): + raise MigrationError("target_escape") + return path.as_posix() + + +def _new_internal_name( + manifest: Mapping[str, Any], entry: Mapping[str, Any], suffix: str +) -> str: + return ( + f".scriptorium-{manifest['batch_id']}-{entry['id']}-" + f"{secrets.token_hex(16)}.{suffix}" + ) + + +def _valid_internal_name( + manifest: Mapping[str, Any], + entry: Mapping[str, Any], + value: Any, + suffix: str, +) -> bool: + if not isinstance(value, str) or Path(value).name != value: + return False + prefix = f".scriptorium-{manifest['batch_id']}-{entry['id']}-" + token, separator, actual_suffix = value.removeprefix(prefix).rpartition(".") + return ( + value.startswith(prefix) + and separator == "." + and actual_suffix == suffix + and _TOKEN_RE.fullmatch(token) is not None + ) + + +def _stage( + manifest: Mapping[str, Any], entry: Mapping[str, Any] +) -> Path | None: + name = entry.get("stage_name") + if name is None: + return None + return Path(entry["target"]).parent / name + + +def _quarantine( + manifest: Mapping[str, Any], entry: Mapping[str, Any] +) -> Path | None: + name = entry.get("quarantine_name") + if name is None: + return None + return Path(entry["target"]).parent / name + + +def _plan_data(manifest: Mapping[str, Any]) -> dict[str, Any]: + fixed = ( + "schema_version", + "privacy", + "batch_id", + "workspace", + "destination_root", + "state_root", + "source_roots", + "limitations", + ) + entry_fixed = ( + "id", + "source", + "target", + "relative_target", + "kind", + "sha256", + "size", + ) + data = {key: manifest.get(key) for key in fixed} + data["entries"] = [ + {key: entry.get(key) for key in entry_fixed} + for entry in manifest.get("entries", []) + ] + return data + + +def _digest(value: Mapping[str, Any]) -> str: + encoded = json.dumps( + value, ensure_ascii=False, sort_keys=True, separators=(",", ":") + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _plan_digest(manifest: Mapping[str, Any]) -> str: + return _digest(_plan_data(manifest)) + + +def _state_digest(manifest: Mapping[str, Any]) -> str: + data = copy.deepcopy(dict(manifest)) + data.pop("state_sha256", None) + return _digest(data) + + +def _seal_manifest(manifest: dict[str, Any], *, new_plan: bool = False) -> None: + if new_plan: + manifest["plan_sha256"] = _plan_digest(manifest) + manifest["state_sha256"] = _state_digest(manifest) + + +def _report( + operation: str, + status: str, + entries: list[Mapping[str, Any]], + *, + roots: int, + changed: int = 0, + unchanged: int = 0, +) -> dict[str, Any]: + return { + "schema_version": REPORT_VERSION, + "operation": operation, + "status": status, + "summary": { + "sources_requested": roots, + "files": len(entries), + "markdown": sum(entry["kind"] == "markdown" for entry in entries), + "pdf": sum(entry["kind"] == "pdf" for entry in entries), + "bytes": sum(entry["size"] for entry in entries), + "changed": changed, + "unchanged": unchanged, + }, + "limitations": list(MIGRATION_LIMITATIONS), + } + + +def format_migration_report(report: Mapping[str, Any]) -> str: + """Render a fixed, path-free aggregate report.""" + + operation = report.get("operation") + if operation not in {"plan", "apply", "load", "verify", "rollback"}: + operation = "migration" + status = report.get("status") + if status not in { + "planned", + "applying", + "applied", + "unchanged", + "rolling-back", + "rolled-back", + }: + status = "unknown" + summary = report.get("summary") + if not isinstance(summary, Mapping): + summary = {} + + def count(name: str) -> int: + value = summary.get(name) + return value if isinstance(value, int) and not isinstance(value, bool) else 0 + + return "\n".join( + ( + f"Migration {operation}: {status}", + ( + f"Files: {count('files')} " + f"(Markdown: {count('markdown')}, PDF: {count('pdf')})" + ), + f"Bytes: {count('bytes')}", + f"Changed: {count('changed')}; unchanged: {count('unchanged')}", + "Private state: canonical local state root; paths suppressed", + ) + ) + + +def plan_migration( + sources: Iterable[os.PathLike[str] | str] | os.PathLike[str] | str, + *, + workspace: os.PathLike[str] | str, + batch_id: str, + state_root: os.PathLike[str] | str | None = None, +) -> MigrationPlan: + """Build a read-only plan from explicit local inputs.""" + + if not isinstance(batch_id, str) or not _BATCH_RE.fullmatch(batch_id): + raise MigrationError("invalid_batch_id") + workspace_path = _existing_directory(workspace, "workspace_missing") + private_root = _private_state_root(state_root, workspace_path) + destination = workspace_path / "Sources" / "Imported" / batch_id + requested = _selected(sources) + entries: list[dict[str, Any]] = [] + seen: set[str] = set() + + for index, source_root in enumerate(requested, start=1): + _reject_remote(source_root) + _check_components(source_root) + metadata = _metadata(source_root) + if metadata is None: + raise MigrationError("source_missing") + if stat.S_ISDIR(metadata.st_mode): + source_root = source_root.resolve(strict=True) + _reject_state_source_overlap(source_root, private_root) + if ( + source_root == destination + or _inside(source_root, destination) + or _inside(destination, source_root) + ): + raise MigrationError("source_destination_overlap") + routes = [ + (source, Path(_label(index, source_root, file=False)) / source.relative_to(source_root)) + for source in _scan(source_root) + ] + elif stat.S_ISREG(metadata.st_mode): + source_root = source_root.resolve(strict=True) + _reject_state_source_overlap(source_root, private_root) + if not _kind(source_root): + raise MigrationError("unsupported_source_type") + if source_root == destination or _inside(destination, source_root): + raise MigrationError("source_destination_overlap") + routes = [(source_root, Path(_label(index, source_root, file=True)))] + else: + raise MigrationError("source_not_regular") + + for source, relative_path in routes: + identity = os.path.normcase(os.path.normpath(os.fspath(source))) + if identity in seen: + continue + seen.add(identity) + relative_target = _relative(relative_path) + target = _absolute(destination / relative_target) + digest, size = _hash_file(source, "source_unreadable") + entry = { + "id": f"file-{len(entries) + 1:06d}", + "source": os.fspath(source), + "target": os.fspath(target), + "relative_target": relative_target, + "kind": _kind(source), + "sha256": digest, + "size": size, + "state": "planned", + "file_identity": None, + "stage_name": None, + "rollback_phase": None, + "quarantine_name": None, + } + _check_components(target.parent) + if _metadata(target) is not None: + raise MigrationError("target_exists") + entries.append(entry) + + if not entries: + raise MigrationError("no_supported_files") + manifest = { + "schema_version": MANIFEST_VERSION, + "privacy": MANIFEST_PRIVACY, + "batch_id": batch_id, + "run_state": "planned", + "workspace": os.fspath(workspace_path), + "destination_root": os.fspath(destination), + "state_root": os.fspath(private_root), + "source_roots": len(requested), + "entries": entries, + "limitations": list(MIGRATION_LIMITATIONS), + } + _seal_manifest(manifest, new_plan=True) + return MigrationPlan( + manifest=manifest, + report=_report("plan", "planned", entries, roots=len(requested)), + ) + + +def _unwrap(value: MigrationPlan | MigrationResult | Mapping[str, Any]) -> dict[str, Any]: + if isinstance(value, (MigrationPlan, MigrationResult)): + value = value.manifest + if not isinstance(value, Mapping): + raise MigrationError("invalid_manifest") + return copy.deepcopy(dict(value)) + + +def _validate_manifest(value: dict[str, Any]) -> dict[str, Any]: + manifest = copy.deepcopy(value) + required = { + "schema_version", "privacy", "batch_id", "run_state", "workspace", + "destination_root", "state_root", "source_roots", "entries", + "limitations", "plan_sha256", "state_sha256", + } + if set(manifest) != required: + raise MigrationError("invalid_manifest") + if ( + manifest["schema_version"] != MANIFEST_VERSION + or manifest["privacy"] != MANIFEST_PRIVACY + or manifest["run_state"] not in _RUN_STATES + or not isinstance(manifest["batch_id"], str) + or not _BATCH_RE.fullmatch(manifest["batch_id"]) + or manifest["limitations"] != list(MIGRATION_LIMITATIONS) + or not isinstance(manifest["source_roots"], int) + or isinstance(manifest["source_roots"], bool) + or manifest["source_roots"] < 1 + or not isinstance(manifest["entries"], list) + or not manifest["entries"] + ): + raise MigrationError("invalid_manifest") + if not all( + isinstance(manifest[key], str) + for key in ("workspace", "destination_root", "state_root") + ): + raise MigrationError("invalid_manifest") + + workspace = _absolute(manifest["workspace"]) + destination = _absolute(manifest["destination_root"]) + state_root = _absolute(manifest["state_root"]) + if destination != workspace / "Sources" / "Imported" / manifest["batch_id"]: + raise MigrationError("target_escape") + if ( + state_root == workspace + or _inside(workspace, state_root) + or _inside(state_root, workspace) + ): + raise MigrationError("state_workspace_overlap") + + entry_keys = { + "id", "source", "target", "relative_target", "kind", "sha256", + "size", "state", "file_identity", "stage_name", "rollback_phase", + "quarantine_name", + } + identifiers: set[str] = set() + targets: set[str] = set() + for entry in manifest["entries"]: + if not isinstance(entry, dict) or set(entry) != entry_keys: + raise MigrationError("invalid_manifest") + if ( + not isinstance(entry["id"], str) + or not _ID_RE.fullmatch(entry["id"]) + or entry["id"] in identifiers + or not isinstance(entry["source"], str) + or not isinstance(entry["target"], str) + or not isinstance(entry["relative_target"], str) + or entry["kind"] not in {"markdown", "pdf"} + or not isinstance(entry["sha256"], str) + or not _HASH_RE.fullmatch(entry["sha256"]) + or not isinstance(entry["size"], int) + or isinstance(entry["size"], bool) + or entry["size"] < 0 + or entry["state"] not in _ENTRY_STATES + ): + raise MigrationError("invalid_manifest") + identity = entry["file_identity"] + if identity is not None and ( + not isinstance(identity, dict) + or set(identity) != {"device", "inode"} + or any( + not isinstance(identity[key], int) + or isinstance(identity[key], bool) + or identity[key] < 0 + for key in ("device", "inode") + ) + or identity["inode"] == 0 + ): + raise MigrationError("invalid_manifest") + stage_name = entry["stage_name"] + phase = entry["rollback_phase"] + quarantine_name = entry["quarantine_name"] + if stage_name is not None and not _valid_internal_name( + manifest, entry, stage_name, "stage" + ): + raise MigrationError("invalid_manifest") + if quarantine_name is not None and not _valid_internal_name( + manifest, entry, quarantine_name, "rollback" + ): + raise MigrationError("invalid_manifest") + if phase is not None and phase not in _ROLLBACK_PHASES: + raise MigrationError("invalid_manifest") + if ( + entry["state"] == "planned" + and any( + value is not None + for value in (identity, stage_name, phase, quarantine_name) + ) + ) or ( + entry["state"] in {"created", "delete-pending"} + and (identity is None or stage_name is None) + ) or ( + entry["state"] != "delete-pending" + and (phase is not None or quarantine_name is not None) + ) or ( + entry["state"] == "delete-pending" + and (phase is None or quarantine_name is None) + ) or ( + (identity is None) != (stage_name is None) + ): + raise MigrationError("invalid_state_transition") + source = Path(entry["source"]) + target = Path(entry["target"]) + relative = _relative(entry["relative_target"]) + expected = _absolute(destination / relative) + key = os.path.normcase(os.path.normpath(os.fspath(target))) + if ( + not source.is_absolute() + or not target.is_absolute() + or target != expected + or not _inside(destination, target) + or _kind(source) != entry["kind"] + or key in targets + ): + raise MigrationError("target_escape") + identifiers.add(entry["id"]) + targets.add(key) + + if ( + not isinstance(manifest["plan_sha256"], str) + or not _HASH_RE.fullmatch(manifest["plan_sha256"]) + or not isinstance(manifest["state_sha256"], str) + or not _HASH_RE.fullmatch(manifest["state_sha256"]) + ): + raise MigrationError("invalid_manifest") + if ( + manifest["plan_sha256"] != _plan_digest(manifest) + or manifest["state_sha256"] != _state_digest(manifest) + ): + raise MigrationError("manifest_integrity_failed") + states = {entry["state"] for entry in manifest["entries"]} + if not states <= _RUN_ENTRY_STATES[manifest["run_state"]]: + raise MigrationError("invalid_state_transition") + return manifest + + +def _workspace_key(workspace: str) -> str: + normalized = os.path.normcase(os.path.normpath(workspace)).encode() + return hashlib.sha256(normalized).hexdigest()[:20] + + +def _manifest_path(manifest: Mapping[str, Any]) -> Path: + return ( + Path(manifest["state_root"]) + / "migrations" + / _workspace_key(manifest["workspace"]) + / f"{manifest['batch_id']}.json" + ) + + +def _lock_path(manifest: Mapping[str, Any]) -> Path: + return _manifest_path(manifest).with_suffix(".lock") + + +def _persist_manifest(manifest: dict[str, Any]) -> None: + _seal_manifest(manifest) + path = _manifest_path(manifest) + _ensure_tree(Path(manifest["state_root"]), path.parent) + payload = ( + json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + ).encode() + descriptor, name = tempfile.mkstemp( + prefix=".migration-", suffix=".tmp", dir=path.parent + ) + temporary = Path(name) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + except OSError as exc: + raise MigrationError("state_write_failed") from exc + finally: + try: + temporary.unlink() + except OSError: + pass + + +def _read_manifest(path: Path) -> dict[str, Any] | None: + metadata = _metadata(path) + if metadata is None: + return None + if _linklike(metadata) or not stat.S_ISREG(metadata.st_mode): + raise MigrationError("stored_manifest_invalid") + try: + value = json.loads(path.read_text(encoding="utf-8")) + stored = _validate_manifest(value) + except (OSError, UnicodeError, json.JSONDecodeError, MigrationError) as exc: + raise MigrationError("stored_manifest_invalid") from exc + return stored + + +def _load_stored(requested: Mapping[str, Any]) -> dict[str, Any] | None: + stored = _read_manifest(_manifest_path(requested)) + if stored is None: + return None + if stored["plan_sha256"] != requested["plan_sha256"]: + raise MigrationError("batch_conflict") + return stored + + +@contextmanager +def _kernel_lock(path: Path): + """Hold an advisory kernel lock released by close or process exit.""" + + _check_components(path.parent) + try: + path.parent.mkdir(parents=True, exist_ok=True) + except OSError as exc: + raise MigrationError("lock_unavailable") from exc + _check_components(path.parent) + metadata = _metadata(path) + if metadata is not None and ( + _linklike(metadata) or not stat.S_ISREG(metadata.st_mode) + ): + raise MigrationError("lock_invalid") + flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_BINARY", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0) + try: + descriptor = os.open(path, flags, 0o600) + handle = os.fdopen(descriptor, "r+b") + except OSError as exc: + raise MigrationError("lock_unavailable") from exc + locked = False + try: + handle.seek(0, os.SEEK_END) + if handle.tell() == 0: + handle.write(b"\0") + handle.flush() + os.fsync(handle.fileno()) + handle.seek(0) + if os.name == "nt": + import msvcrt + + msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1) + else: + import fcntl + + fcntl.flock(handle.fileno(), fcntl.LOCK_EX) + locked = True + yield + except OSError as exc: + raise MigrationError("lock_unavailable") from exc + finally: + if locked: + handle.seek(0) + if os.name == "nt": + msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) + else: + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + handle.close() + + +def _stored_identity( + workspace: os.PathLike[str] | str, + batch_id: str, +) -> tuple[Path, dict[str, str]]: + if not isinstance(batch_id, str) or not _BATCH_RE.fullmatch(batch_id): + raise MigrationError("invalid_batch_id") + workspace_path = _existing_directory(workspace, "workspace_missing") + state_root = _private_state_root(None, workspace_path) + return workspace_path, { + "workspace": os.fspath(workspace_path), + "state_root": os.fspath(state_root), + "batch_id": batch_id, + } + + +def load_migration( + *, + workspace: os.PathLike[str] | str, + batch_id: str, +) -> MigrationResult: + """Load private migration state using only its stable public identity.""" + + workspace_path, identity = _stored_identity(workspace, batch_id) + stored = _read_manifest(_manifest_path(identity)) + if stored is None: + raise MigrationError("migration_not_found") + if ( + Path(stored["workspace"]) != workspace_path + or stored["batch_id"] != batch_id + or not _same_path(Path(stored["state_root"]), Path(identity["state_root"])) + ): + raise MigrationError("stored_manifest_invalid") + return MigrationResult( + stored, + _report( + "load", + stored["run_state"], + stored["entries"], + roots=stored["source_roots"], + unchanged=len(stored["entries"]), + ), + ) + + +def _content_status(path: Path | None, entry: Mapping[str, Any]) -> str: + if path is None: + return "missing" + metadata = _metadata(path) + if metadata is None: + return "missing" + if _linklike(metadata) or not stat.S_ISREG(metadata.st_mode): + return "changed" + digest, size = _hash_file(path, "path_unreadable") + return "exact" if (digest, size) == (entry["sha256"], entry["size"]) else "changed" + + +def _identity_from_metadata(metadata: os.stat_result) -> dict[str, int]: + """Return a stable local identity from already-bound file metadata.""" + + if _linklike(metadata) or not stat.S_ISREG(metadata.st_mode): + raise MigrationError("file_identity_unavailable") + device = getattr(metadata, "st_dev", None) + inode = getattr(metadata, "st_ino", None) + if ( + not isinstance(device, int) + or isinstance(device, bool) + or device < 0 + or not isinstance(inode, int) + or isinstance(inode, bool) + or inode <= 0 + ): + raise MigrationError("file_identity_unavailable") + return {"device": device, "inode": inode} + + +def _file_identity(path: Path | None) -> dict[str, int] | None: + """Return a stable local file identity or fail closed when unavailable.""" + + if path is None: + return None + metadata = _metadata(path) + if metadata is None: + return None + return _identity_from_metadata(metadata) + + +def _identity_matches(path: Path | None, entry: Mapping[str, Any]) -> bool: + expected = entry.get("file_identity") + return isinstance(expected, Mapping) and _file_identity(path) == dict(expected) + + +def _require_owned_anchor( + manifest: Mapping[str, Any], entry: Mapping[str, Any] +) -> None: + anchor = _stage(manifest, entry) + if ( + _content_status(anchor, entry) != "exact" + or not _identity_matches(anchor, entry) + ): + raise MigrationError("owned_target_changed") + + +def _require_owned_target( + manifest: Mapping[str, Any], entry: Mapping[str, Any] +) -> None: + """Require the published path to remain the anchor's unchanged hard link.""" + + target = Path(entry["target"]) + anchor = _stage(manifest, entry) + if ( + _content_status(target, entry) != "exact" + or _content_status(anchor, entry) != "exact" + or not _identity_matches(target, entry) + or not _identity_matches(anchor, entry) + ): + raise MigrationError("owned_target_changed") + try: + same_file = os.path.samefile(anchor, target) + except OSError as exc: + raise MigrationError("owned_target_changed") from exc + if not same_file: + raise MigrationError("owned_target_changed") + + +def _verify_source(entry: Mapping[str, Any]) -> None: + actual = _hash_file(Path(entry["source"]), "source_missing") + if actual != (entry["sha256"], entry["size"]): + raise MigrationError("source_hash_changed") + + +def _copy_stage(manifest: dict[str, Any], entry: dict[str, Any]) -> Path: + """Create or recover one random, manifest-owned same-directory stage.""" + + stage = _stage(manifest, entry) + if stage is not None: + _require_owned_anchor(manifest, entry) + return stage + if entry.get("file_identity") is not None: + raise MigrationError("owned_target_changed") + _verify_source(entry) + target_parent = Path(entry["target"]).parent + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0) + for _ in range(32): + name = _new_internal_name(manifest, entry, "stage") + candidate = target_parent / name + try: + descriptor = os.open(candidate, flags, 0o600) + except FileExistsError: + # A same-name entry is never adopted or removed. + continue + except OSError as exc: + raise MigrationError("target_unwritable") from exc + + digest = hashlib.sha256() + size = 0 + identity = None + try: + with os.fdopen(descriptor, "wb") as target: + with Path(entry["source"]).open("rb") as source: + while chunk := source.read(1024 * 1024): + target.write(chunk) + digest.update(chunk) + size += len(chunk) + target.flush() + os.fsync(target.fileno()) + identity = _identity_from_metadata(os.fstat(target.fileno())) + except OSError as exc: + # The random entry may no longer be ours after the handle closes. + # Preserve it for manual inspection instead of unlinking by path. + raise MigrationError("target_unwritable") from exc + if (digest.hexdigest(), size) != (entry["sha256"], entry["size"]): + # Never unlink a path whose ownership was not durably recorded. + raise MigrationError("source_hash_changed") + if identity is None: + raise MigrationError("file_identity_unavailable") + + entry["stage_name"] = name + entry["file_identity"] = identity + _persist_manifest(manifest) + _require_owned_anchor(manifest, entry) + return candidate + raise MigrationError("staging_conflict") + + +def _publish(stage: Path, target: Path) -> None: + if _metadata(target) is not None: + raise MigrationError("target_exists") + try: + os.link(stage, target) + except FileExistsError as exc: + raise MigrationError("target_exists") from exc + except OSError as exc: + code = ( + "cross_volume_publish_unsupported" + if exc.errno == errno.EXDEV + else "atomic_publish_unavailable" + ) + raise MigrationError(code) from exc + + +def _preflight_apply(manifest: Mapping[str, Any], *, new: bool) -> None: + for entry in manifest["entries"]: + state = entry["state"] + target = _content_status(Path(entry["target"]), entry) + stage = _content_status(_stage(manifest, entry), entry) + if new or state == "planned": + if target != "missing": + raise MigrationError("target_exists") + _verify_source(entry) + elif state == "created": + _require_owned_target(manifest, entry) + elif state == "creating": + if target == "changed": + raise MigrationError("owned_target_changed") + if target == "exact": + _require_owned_target(manifest, entry) + elif stage == "changed": + raise MigrationError("staging_conflict") + elif stage == "exact": + _require_owned_anchor(manifest, entry) + elif entry.get("stage_name") is not None: + raise MigrationError("owned_target_changed") + else: + _verify_source(entry) + else: + raise MigrationError("rollback_in_progress") + + +def apply_migration( + plan: MigrationPlan | MigrationResult | Mapping[str, Any], +) -> MigrationResult: + """Apply a plan with atomic create-if-absent publication.""" + + requested = _validate_manifest(_unwrap(plan)) + workspace = _existing_directory(requested["workspace"], "workspace_missing") + if workspace != Path(requested["workspace"]): + raise MigrationError("workspace_identity_changed") + _private_state_root(requested["state_root"], workspace) + _ensure_tree(Path(requested["state_root"]), _lock_path(requested).parent) + + with _kernel_lock(_lock_path(requested)): + stored = _load_stored(requested) + if stored is None: + if requested["run_state"] != "planned": + raise MigrationError("stored_manifest_missing") + current = requested + _preflight_apply(current, new=True) + current["run_state"] = "applying" + _persist_manifest(current) + else: + current = stored + if current["run_state"] == "applied": + _preflight_apply(current, new=False) + return MigrationResult( + current, + _report( + "apply", "unchanged", current["entries"], + roots=current["source_roots"], + unchanged=len(current["entries"]), + ), + ) + if current["run_state"] == "rolled-back": + raise MigrationError("migration_already_rolled_back") + if current["run_state"] == "rolling-back": + raise MigrationError("rollback_in_progress") + _preflight_apply(current, new=False) + + changed = 0 + for entry in current["entries"]: + if entry["state"] == "created": + continue + target = Path(entry["target"]) + _ensure_tree(workspace, target.parent) + if entry["state"] == "creating" and _content_status(target, entry) == "exact": + _require_owned_target(current, entry) + entry["state"] = "created" + _persist_manifest(current) + continue + if entry["state"] == "planned": + entry["state"] = "creating" + _persist_manifest(current) + stage = _copy_stage(current, entry) + _publish(stage, target) + _require_owned_target(current, entry) + entry["state"] = "created" + changed += 1 + _persist_manifest(current) + + current["run_state"] = "applied" + _persist_manifest(current) + return MigrationResult( + current, + _report( + "apply", "applied", current["entries"], + roots=current["source_roots"], + changed=changed, + unchanged=len(current["entries"]) - changed, + ), + ) + + +def _rename_noreplace(source: Path, target: Path) -> None: + """Atomically move one path without replacing an existing destination.""" + + if os.name == "nt": + try: + os.rename(source, target) + return + except FileExistsError as exc: + raise MigrationError("quarantine_conflict") from exc + except OSError as exc: + if getattr(exc, "winerror", None) in {80, 183}: + raise MigrationError("quarantine_conflict") from exc + raise MigrationError("rollback_failed") from exc + + if sys.platform.startswith("linux"): + libc = ctypes.CDLL(None, use_errno=True) + renameat2 = getattr(libc, "renameat2", None) + if renameat2 is None: + raise MigrationError("atomic_quarantine_unavailable") + renameat2.argtypes = [ + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_uint, + ] + renameat2.restype = ctypes.c_int + result = renameat2( + _AT_FDCWD, + os.fsencode(source), + _AT_FDCWD, + os.fsencode(target), + _RENAME_NOREPLACE, + ) + if result == 0: + return + error = ctypes.get_errno() + if error in {errno.EEXIST, errno.ENOTEMPTY}: + raise MigrationError("quarantine_conflict") + if error in { + errno.ENOSYS, + errno.EINVAL, + getattr(errno, "EOPNOTSUPP", errno.EINVAL), + }: + raise MigrationError("atomic_quarantine_unavailable") + raise MigrationError("rollback_failed") + + raise MigrationError("atomic_quarantine_unavailable") + + +def _require_owned_path(path: Path | None, entry: Mapping[str, Any]) -> None: + if ( + path is None + or _content_status(path, entry) != "exact" + or not _identity_matches(path, entry) + ): + raise MigrationError("owned_target_changed") + + +def _has_owned_identity(path: Path, entry: Mapping[str, Any]) -> bool: + try: + return _identity_matches(path, entry) + except MigrationError as exc: + if exc.code == "file_identity_unavailable": + return False + raise + + +def _restore_unowned_quarantine(quarantine: Path, source: Path) -> None: + if _metadata(source) is not None: + raise MigrationError("rollback_restore_blocked") + try: + _rename_noreplace(quarantine, source) + except MigrationError as exc: + if exc.code in {"quarantine_conflict", "rollback_failed"}: + raise MigrationError("rollback_restore_blocked") from exc + raise + raise MigrationError("owned_target_changed") + + +def _move_owned_to_quarantine( + manifest: Mapping[str, Any], + entry: Mapping[str, Any], + source: Path, +) -> None: + quarantine = _quarantine(manifest, entry) + if quarantine is None: + raise MigrationError("invalid_state_transition") + source_exists = _metadata(source) is not None + quarantine_exists = _metadata(quarantine) is not None + if quarantine_exists and source_exists: + raise MigrationError("quarantine_conflict") + if not quarantine_exists: + if not source_exists: + return + _rename_noreplace(source, quarantine) + quarantine_exists = True + if not quarantine_exists: + return + if ( + _content_status(quarantine, entry) != "exact" + or not _identity_matches(quarantine, entry) + ): + _restore_unowned_quarantine(quarantine, source) + if _metadata(source) is not None and _has_owned_identity(source, entry): + raise MigrationError("owned_target_changed") + + +def _delete_owned_quarantine( + manifest: Mapping[str, Any], entry: Mapping[str, Any] +) -> None: + quarantine = _quarantine(manifest, entry) + if quarantine is None or _metadata(quarantine) is None: + return + _require_owned_path(quarantine, entry) + try: + quarantine.unlink() + except OSError as exc: + raise MigrationError("rollback_failed") from exc + + +def _set_rollback_phase( + manifest: dict[str, Any], + entry: dict[str, Any], + phase: str, +) -> None: + entry["rollback_phase"] = phase + entry["quarantine_name"] = _new_internal_name(manifest, entry, "rollback") + _persist_manifest(manifest) + + +def _preflight_pending_rollback( + manifest: Mapping[str, Any], + entry: Mapping[str, Any], + *, + allow_recovery: bool, +) -> None: + phase = entry["rollback_phase"] + quarantine = _quarantine(manifest, entry) + quarantine_status = _content_status(quarantine, entry) + target = Path(entry["target"]) + anchor = _stage(manifest, entry) + + if phase == "target": + if quarantine_status == "exact" and _identity_matches(quarantine, entry): + _require_owned_anchor(manifest, entry) + return + if quarantine_status != "missing": + if allow_recovery: + return + raise MigrationError("owned_target_changed") + target_status = _content_status(target, entry) + if target_status == "exact": + _require_owned_target(manifest, entry) + elif target_status == "changed": + if not allow_recovery: + raise MigrationError("owned_target_changed") + else: + _require_owned_anchor(manifest, entry) + return + + if phase == "target-quarantined": + if quarantine_status != "missing": + _require_owned_path(quarantine, entry) + _require_owned_anchor(manifest, entry) + return + + if phase == "anchor": + if quarantine_status == "exact" and _identity_matches(quarantine, entry): + return + if quarantine_status != "missing": + if allow_recovery: + return + raise MigrationError("owned_target_changed") + if anchor is not None and _metadata(anchor) is not None: + _require_owned_anchor(manifest, entry) + return + + if phase == "anchor-quarantined": + if quarantine_status != "missing": + _require_owned_path(quarantine, entry) + return + + raise MigrationError("invalid_state_transition") + + +def _preflight_rollback( + manifest: Mapping[str, Any], *, allow_recovery: bool = False +) -> None: + for entry in manifest["entries"]: + state = entry["state"] + target = _content_status(Path(entry["target"]), entry) + anchor = _content_status(_stage(manifest, entry), entry) + if state == "created": + if target == "exact": + _require_owned_target(manifest, entry) + elif target == "missing": + _require_owned_anchor(manifest, entry) + else: + raise MigrationError("owned_target_changed") + elif state == "creating": + if entry["stage_name"] is None: + if target != "missing": + raise MigrationError("target_exists") + continue + if target == "changed": + raise MigrationError("owned_target_changed") + if target == "exact": + _require_owned_target(manifest, entry) + elif anchor == "exact": + _require_owned_anchor(manifest, entry) + elif anchor == "changed": + raise MigrationError("owned_target_changed") + elif state == "delete-pending": + _preflight_pending_rollback( + manifest, entry, allow_recovery=allow_recovery + ) + elif state == "planned": + if target != "missing": + raise MigrationError("target_exists") + + +def verify_migration( + *, + workspace: os.PathLike[str] | str, + batch_id: str, +) -> MigrationResult: + """Verify stored state and owned files without needing the original plan.""" + + loaded = load_migration(workspace=workspace, batch_id=batch_id) + requested = loaded.manifest + _ensure_tree(Path(requested["state_root"]), _lock_path(requested).parent) + with _kernel_lock(_lock_path(requested)): + current = _load_stored(requested) + if current is None: + raise MigrationError("migration_not_found") + if current["run_state"] == "planned": + _preflight_apply(current, new=True) + elif current["run_state"] in {"applying", "applied"}: + _preflight_apply(current, new=False) + else: + _preflight_rollback(current) + return MigrationResult( + current, + _report( + "verify", + current["run_state"], + current["entries"], + roots=current["source_roots"], + unchanged=len(current["entries"]), + ), + ) + + +def rollback_migration( + applied: MigrationPlan | MigrationResult | Mapping[str, Any], +) -> MigrationResult: + """Remove unchanged owned files with retryable per-entry state.""" + + requested = _validate_manifest(_unwrap(applied)) + workspace = _existing_directory(requested["workspace"], "workspace_missing") + if workspace != Path(requested["workspace"]): + raise MigrationError("workspace_identity_changed") + _private_state_root(requested["state_root"], workspace) + _ensure_tree(Path(requested["state_root"]), _lock_path(requested).parent) + + with _kernel_lock(_lock_path(requested)): + current = _load_stored(requested) + if current is None: + raise MigrationError("applied_manifest_missing") + if current["run_state"] == "rolled-back": + return MigrationResult( + current, + _report( + "rollback", "unchanged", current["entries"], + roots=current["source_roots"], + unchanged=len(current["entries"]), + ), + ) + if current["run_state"] not in {"applying", "applied", "rolling-back"}: + raise MigrationError("migration_not_applied") + _preflight_rollback(current, allow_recovery=True) + if current["run_state"] != "rolling-back": + current["run_state"] = "rolling-back" + _persist_manifest(current) + + changed = 0 + for entry in current["entries"]: + state = entry["state"] + target = Path(entry["target"]) + if state == "deleted": + continue + if state == "planned": + entry["state"] = "deleted" + _persist_manifest(current) + continue + if state == "creating" and entry["stage_name"] is None: + entry["state"] = "deleted" + _persist_manifest(current) + continue + if state != "delete-pending": + entry["state"] = "delete-pending" + _set_rollback_phase(current, entry, "target") + + while entry["state"] == "delete-pending": + phase = entry["rollback_phase"] + if phase == "target": + _move_owned_to_quarantine(current, entry, target) + entry["rollback_phase"] = "target-quarantined" + _persist_manifest(current) + continue + if phase == "target-quarantined": + _delete_owned_quarantine(current, entry) + _set_rollback_phase(current, entry, "anchor") + continue + if phase == "anchor": + anchor = _stage(current, entry) + if anchor is None: + raise MigrationError("invalid_state_transition") + _move_owned_to_quarantine(current, entry, anchor) + entry["rollback_phase"] = "anchor-quarantined" + _persist_manifest(current) + continue + if phase == "anchor-quarantined": + _delete_owned_quarantine(current, entry) + entry["state"] = "deleted" + entry["rollback_phase"] = None + entry["quarantine_name"] = None + changed += 1 + _persist_manifest(current) + continue + raise MigrationError("invalid_state_transition") + + current["run_state"] = "rolled-back" + _persist_manifest(current) + return MigrationResult( + current, + _report( + "rollback", "rolled-back", current["entries"], + roots=current["source_roots"], + changed=changed, + unchanged=len(current["entries"]) - changed, + ), + ) + + +def reapply_migration( + *, + workspace: os.PathLike[str] | str, + batch_id: str, +) -> MigrationResult: + """Resume or repeat apply using only workspace and batch identity.""" + + loaded = load_migration(workspace=workspace, batch_id=batch_id) + return apply_migration(loaded) + + +__all__ = [ + "MANIFEST_VERSION", + "MIGRATION_LIMITATIONS", + "MIGRATION_ERROR_CODES", + "MigrationError", + "MigrationPlan", + "MigrationResult", + "apply_migration", + "format_migration_report", + "load_migration", + "plan_migration", + "reapply_migration", + "rollback_migration", + "verify_migration", +] diff --git a/src/scriptorium/resume.py b/src/scriptorium/resume.py new file mode 100644 index 0000000..1fee4a0 --- /dev/null +++ b/src/scriptorium/resume.py @@ -0,0 +1,462 @@ +"""Read-only Scriptorium entry for a bounded Provenance context capsule.""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +from pathlib import Path +from typing import Any + +from . import __version__ +from .demo import DemoError, find_script, load_compatibility, project_version, resolve_component_root +from .path_selection import format_path_selection + + +PUBLIC_COMMAND = "prov-context" +RESUME_TIMEOUT_SECONDS = 30 +CAPSULE_VERSION = "context-capsule/0.1" +ARTIFACT_KINDS = {"parsed-paper", "reading-note", "review", "lineage-graph"} +ARTIFACT_SCHEMAS = { + "parsed-paper/1.0", + "reading-note/1.0", + "review/1.0", + "lineage-graph/1.0", +} +PASSTHROUGH_ENV_NAMES = { + "APPDATA", + "COMSPEC", + "HOME", + "LANG", + "LC_ALL", + "LC_CTYPE", + "LOCALAPPDATA", + "PATH", + "PATHEXT", + "SYSTEMDRIVE", + "SYSTEMROOT", + "TEMP", + "TMP", + "TMPDIR", + "USERPROFILE", + "WINDIR", +} +ENTRY_LIMITATIONS = ( + "The capsule contains human-authored or approved project state only.", + "Literature and research artifacts remain reference-only, not approved scientific claims.", + "The capsule is local research context and must be reviewed before sharing.", +) +_WINDOWS_PATH = re.compile(r"(?i)(?:[a-z]:[\\/]|\\\\[^\\\s]+[\\/])") +_PRIVATE_POSIX_PATH = re.compile(r"(?:^|[\s\"'])(?:/(?:home|Users|root|mnt|private|tmp)/)") + + +class ResumeError(RuntimeError): + """A trustworthy context capsule could not be produced.""" + + +def _resolve_directory(path: Path, *, label: str) -> Path: + try: + resolved = path.expanduser().resolve(strict=True) + except (OSError, RuntimeError) as exc: + raise ResumeError(f"{label} does not exist or cannot be resolved") from exc + if not resolved.is_dir(): + raise ResumeError(f"{label} is not a directory") + return resolved + + +def _compatibility_version() -> str: + try: + version = load_compatibility()["provenance"] + except (DemoError, KeyError, OSError, TypeError, ValueError) as exc: + raise ResumeError("packaged Provenance compatibility data is unavailable") from exc + if not version: + raise ResumeError("packaged Provenance compatibility version is empty") + return version + + +def _resolve_provenance( + explicit: Path | None, *, expected_version: str +) -> tuple[Path | None, Path]: + root: Path | None = None + if explicit is not None: + root = _resolve_directory(explicit, label="Provenance root") + else: + try: + root = resolve_component_root("provenance") + except (DemoError, OSError, RuntimeError) as exc: + configured = os.environ.get("SCRIPTORIUM_PROVENANCE_ROOT") + if configured and configured.strip(): + raise ResumeError("configured Provenance root is unavailable") from exc + + if root is not None: + try: + actual_version = project_version(root) + except DemoError as exc: + raise ResumeError("Provenance source metadata is unavailable") from exc + if actual_version != expected_version: + raise ResumeError( + "incompatible Provenance source version: " + f"expected {expected_version}, found {actual_version}" + ) + try: + return root, find_script(root, PUBLIC_COMMAND) + except DemoError as exc: + raise ResumeError(f"public command '{PUBLIC_COMMAND}' is unavailable") from exc + + installed = shutil.which(PUBLIC_COMMAND) + if not installed: + raise ResumeError(f"public command '{PUBLIC_COMMAND}' is unavailable") + return None, Path(installed).resolve() + + +def _resume_environment(provenance_home: Path) -> dict[str, str]: + env = { + name: value + for name, value in os.environ.items() + if name.upper() in PASSTHROUGH_ENV_NAMES + } + env.update( + { + "PROVENANCE_HOME": str(provenance_home), + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONIOENCODING": "utf-8", + "PYTHONNOUSERSITE": "1", + "PYTHONUTF8": "1", + } + ) + return env + + +def _object(value: Any, *, fields: set[str], label: str) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != fields: + raise ResumeError(f"context capsule field '{label}' is incompatible") + return value + + +def _text(value: Any, *, label: str, maximum: int = 4000) -> str: + if not isinstance(value, str) or len(value) > maximum: + raise ResumeError(f"context capsule field '{label}' is incompatible") + if _WINDOWS_PATH.search(value) or _PRIVATE_POSIX_PATH.search(value): + raise ResumeError("context capsule contains a suppressed local path") + return value + + +def _text_list( + value: Any, *, label: str, maximum_items: int = 100, maximum_text: int = 2000 +) -> list[str]: + if not isinstance(value, list) or len(value) > maximum_items: + raise ResumeError(f"context capsule field '{label}' is incompatible") + return [ + _text(item, label=f"{label}[]", maximum=maximum_text) for item in value + ] + + +def _rebuild_project(value: Any) -> dict[str, str]: + fields = { + "project_id", + "title", + "status", + "stage", + "priority", + "updated", + "goal", + "conclusion", + } + project = _object(value, fields=fields, label="project") + return { + key: _text(project[key], label=f"project.{key}") + for key in ( + "project_id", + "title", + "status", + "stage", + "priority", + "updated", + "goal", + "conclusion", + ) + } + + +def _rebuild_recent_progress(value: Any) -> list[dict[str, Any]]: + if not isinstance(value, list) or len(value) > 30: + raise ResumeError("context capsule field 'recent_progress' is incompatible") + rebuilt = [] + for item in value: + entry = _object(item, fields={"date", "items"}, label="recent_progress[]") + rebuilt.append( + { + "date": _text(entry["date"], label="recent_progress[].date", maximum=64), + "items": _text_list( + entry["items"], label="recent_progress[].items", maximum_items=30 + ), + } + ) + return rebuilt + + +def _rebuild_literature(value: Any) -> list[dict[str, Any]]: + if not isinstance(value, list) or len(value) > 100: + raise ResumeError("context capsule field 'literature' is incompatible") + rebuilt = [] + fields = {"id", "citekey", "title", "year", "read_status", "tldr"} + for item in value: + entry = _object(item, fields=fields, label="literature[]") + year = entry["year"] + if year is not None and ( + isinstance(year, bool) or not isinstance(year, (int, str)) + ): + raise ResumeError("context capsule field 'literature[].year' is incompatible") + rebuilt.append( + { + "id": _text(entry["id"], label="literature[].id", maximum=256), + "citekey": _text( + entry["citekey"], label="literature[].citekey", maximum=256 + ), + "title": _text(entry["title"], label="literature[].title"), + "year": year, + "read_status": _text( + entry["read_status"], label="literature[].read_status", maximum=128 + ), + "tldr": _text(entry["tldr"], label="literature[].tldr"), + } + ) + return rebuilt + + +def _rebuild_artifacts(value: Any) -> list[dict[str, Any]]: + if not isinstance(value, list) or len(value) > 100: + raise ResumeError("context capsule field 'research_artifacts' is incompatible") + rebuilt = [] + fields = { + "kind", + "schema_version", + "id", + "title", + "status", + "source_ids", + "summary", + "trust", + } + for item in value: + entry = _object(item, fields=fields, label="research_artifacts[]") + kind = _text(entry["kind"], label="research_artifacts[].kind", maximum=64) + schema_version = _text( + entry["schema_version"], + label="research_artifacts[].schema_version", + maximum=64, + ) + if kind not in ARTIFACT_KINDS or schema_version not in ARTIFACT_SCHEMAS: + raise ResumeError("context capsule research artifact type is incompatible") + if entry["trust"] != "reference_only": + raise ResumeError("context capsule research artifact trust is incompatible") + rebuilt.append( + { + "kind": kind, + "schema_version": schema_version, + "id": _text(entry["id"], label="research_artifacts[].id", maximum=256), + "title": _text(entry["title"], label="research_artifacts[].title"), + "status": _text( + entry["status"], label="research_artifacts[].status", maximum=128 + ), + "source_ids": _text_list( + entry["source_ids"], + label="research_artifacts[].source_ids", + maximum_items=100, + maximum_text=256, + ), + "summary": _text(entry["summary"], label="research_artifacts[].summary"), + "trust": "reference_only", + } + ) + return rebuilt + + +def _parse_capsule(stdout: str) -> dict[str, Any]: + try: + value = json.loads(stdout) + except json.JSONDecodeError as exc: + raise ResumeError("public context command returned non-JSON or extra stdout") from exc + fields = { + "capsule_version", + "project", + "next_actions", + "blocked_by", + "recent_progress", + "literature", + "research_artifacts", + "reference_leads", + "trust", + "limits", + } + capsule = _object(value, fields=fields, label="root") + if capsule["capsule_version"] != CAPSULE_VERSION: + raise ResumeError("public context command returned an incompatible capsule version") + + leads = _object( + capsule["reference_leads"], + fields={"gaps", "priority_reads"}, + label="reference_leads", + ) + trust = _object( + capsule["trust"], + fields={"project_state", "recent_progress", "research_artifacts"}, + label="trust", + ) + if trust != { + "project_state": "human_or_approved", + "recent_progress": "auto_applied_low_risk_not_approved_claims", + "research_artifacts": "reference_only_not_approved_claims", + }: + raise ResumeError("public context command returned incompatible trust semantics") + limits = _object( + capsule["limits"], fields={"max_markdown_chars", "truncated"}, label="limits" + ) + if limits.get("max_markdown_chars") != 8000 or not isinstance( + limits.get("truncated"), bool + ): + raise ResumeError("public context command returned incompatible limits") + + return { + "capsule_version": CAPSULE_VERSION, + "project": _rebuild_project(capsule["project"]), + "next_actions": _text_list(capsule["next_actions"], label="next_actions"), + "blocked_by": _text(capsule["blocked_by"], label="blocked_by"), + "recent_progress": _rebuild_recent_progress(capsule["recent_progress"]), + "literature": _rebuild_literature(capsule["literature"]), + "research_artifacts": _rebuild_artifacts(capsule["research_artifacts"]), + "reference_leads": { + "gaps": _text_list(leads["gaps"], label="reference_leads.gaps"), + "priority_reads": _text_list( + leads["priority_reads"], label="reference_leads.priority_reads" + ), + }, + "trust": dict(trust), + "limits": dict(limits), + } + + +def run_resume( + *, provenance_home: Path, project: str, provenance_root: Path | None = None +) -> dict[str, Any]: + """Read one project capsule through Provenance's public command.""" + resolved_home = _resolve_directory(provenance_home, label="Provenance home") + expected_version = _compatibility_version() + resolved_root, command = _resolve_provenance( + provenance_root, expected_version=expected_version + ) + if not isinstance(project, str) or not project.strip(): + raise ResumeError("project is required") + cwd = resolved_root or resolved_home + environment = _resume_environment(resolved_home) + try: + version_probe = subprocess.run( + [str(command), "--version"], + cwd=cwd, + env=environment, + capture_output=True, + text=True, + encoding="utf-8", + errors="strict", + shell=False, + timeout=RESUME_TIMEOUT_SECONDS, + check=False, + ) + if ( + version_probe.returncode != 0 + or version_probe.stdout.strip() != expected_version + ): + raise ResumeError( + f"public command '{PUBLIC_COMMAND}' has an incompatible runtime version" + ) + argv = [str(command), "--project", project, "--json"] + completed = subprocess.run( + argv, + cwd=cwd, + env=environment, + capture_output=True, + text=True, + encoding="utf-8", + errors="strict", + shell=False, + timeout=RESUME_TIMEOUT_SECONDS, + check=False, + ) + except subprocess.TimeoutExpired as exc: + raise ResumeError(f"public command '{PUBLIC_COMMAND}' timed out") from exc + except (OSError, UnicodeError) as exc: + raise ResumeError(f"public command '{PUBLIC_COMMAND}' could not run") from exc + if completed.returncode != 0: + raise ResumeError(f"public command '{PUBLIC_COMMAND}' did not return a capsule") + + capsule = _parse_capsule(completed.stdout) + return { + "format_version": 1, + "generated_by": {"name": "scriptorium", "version": __version__}, + "operation": "resume", + "status": "ready", + "exit_code": 0, + "capsule": capsule, + "egress": { + "suite_managed": "not-requested", + "host_managed": "not-invoked", + "optional_connectors": "not-invoked", + }, + "entry": { + "public_command": PUBLIC_COMMAND, + "component_generated_by": { + "name": "provenance", + "version": expected_version, + }, + "expected_component_version": expected_version, + "component_exit_code": completed.returncode, + "stdout": "parsed-and-suppressed", + "stderr": "suppressed" if completed.stderr else "empty", + }, + "limitations": list(ENTRY_LIMITATIONS), + } + + +def format_resume_report(report: dict[str, Any]) -> str: + """Render the trusted capsule without exposing component diagnostics or paths.""" + capsule = report["capsule"] + project = capsule["project"] + lines = [ + f"Scriptorium resume {report['generated_by']['version']}", + f"Project: {project['title']} ({project['project_id']})", + f"State: {project['status']} / {project['stage']}", + ] + if project["goal"]: + lines.extend(["", "Goal:", project["goal"]]) + if project["conclusion"]: + lines.extend(["", "Approved conclusion:", project["conclusion"]]) + if capsule["next_actions"]: + lines.extend(["", "Next actions:"]) + lines.extend(f"- {item}" for item in capsule["next_actions"]) + if capsule["blocked_by"]: + lines.extend(["", "Blocked by:", capsule["blocked_by"]]) + if capsule["recent_progress"]: + lines.extend(["", "Recent progress (auto-applied low-risk context):"]) + for entry in capsule["recent_progress"]: + lines.extend(f"- {entry['date']}: {item}" for item in entry["items"]) + if capsule["literature"] or capsule["research_artifacts"]: + lines.extend( + [ + "", + "Reference context:", + f"- Literature records: {len(capsule['literature'])}", + f"- Research artifacts: {len(capsule['research_artifacts'])}", + "- Trust: reference-only; verify against primary evidence.", + ] + ) + if capsule["limits"]["truncated"]: + lines.extend(["", "Note: the bounded capsule was truncated."]) + lines.extend( + format_path_selection( + report, + conflict_guidance="pass an explicit --provenance-home to select the intended local data root.", + ) + ) + return "\n".join(lines) diff --git a/tests/_e2e_lectern_worker.py b/tests/_e2e_lectern_worker.py new file mode 100644 index 0000000..2622794 --- /dev/null +++ b/tests/_e2e_lectern_worker.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 +"""Lectern-side worker for the synthetic cross-repository golden path.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +from asa_agents import FakeLLM, build_outline +from asa_agents.graph import build_graph +from ingestion import ingest_handoff +from langgraph.types import Command +from pptx import Presentation +from slide_ir import GenerationState + + +class WorkerFailure(RuntimeError): + """Lectern violated a golden-path invariant.""" + + +def require(condition: bool, message: str) -> None: + if not condition: + raise WorkerFailure(message) + + +def read_json(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + require(isinstance(value, dict), "Handoff metadata must be an object") + return value + + +def scripted_deck(meta: dict[str, Any]) -> str: + papers = meta["papers"] + rows = [ + [ + str(paper["title"]), + f"SYNTHETIC-PAPER-{index}-EVIDENCE", + str(paper["doi"]), + ] + for index, paper in enumerate(papers, start=1) + ] + return json.dumps( + { + "deck_id": "synthetic-lectern-golden", + "slides": [ + { + "slide_id": "cover", + "layout_type": "title", + "title": "[SYNTHETIC] Dual-paper evidence review", + "subtitle": "Offline, credential-free golden path", + "blocks": [], + "speaker_notes": "This deck contains synthetic evidence only.", + "provenance": {"source": "report_basis"}, + }, + { + "slide_id": "evidence", + "layout_type": "two_column_table", + "title": "Evidence-backed comparison", + "blocks": [ + { + "type": "table", + "columns": ["Source", "Evidence marker", "DOI"], + "rows": rows, + "needs_human_check": False, + } + ], + "speaker_notes": "Each row remains traceable to the handoff.", + "provenance": { + "source": f"handoff:{meta['key']}", + "paper_count": len(papers), + }, + }, + ], + }, + ensure_ascii=True, + ) + + +def all_text(prs: Presentation) -> list[str]: + values: list[str] = [] + for slide in prs.slides: + for shape in slide.shapes: + if getattr(shape, "has_text_frame", False): + values.append(shape.text) + if getattr(shape, "has_table", False): + table = shape.table + for row in table.rows: + values.extend(cell.text for cell in row.cells) + return values + + +def field(value: Any, name: str, default: Any = None) -> Any: + if isinstance(value, dict): + return value.get(name, default) + return getattr(value, name, default) + + +def run(handoff: Path, output_dir: Path) -> dict[str, Any]: + meta = read_json(handoff / "meta.json") + papers = meta.get("papers") + require(meta.get("schema_version") == "handoff/1.1", "Expected handoff/1.1") + require(isinstance(papers, list) and len(papers) == 2, "Expected two papers") + + output_dir.mkdir(parents=True, exist_ok=True) + require(not list(output_dir.rglob("*.pptx")), "Output directory was not empty") + + ingested = ingest_handoff(handoff) + evidence_text = "\n".join( + str(asset.content_ref) + for asset in ingested.assets + if str(asset.kind.value) == "section_text" + ) + expected_evidence = [ + str(paper[field]) + for paper in papers + for field in ("title", "doi") + ] + [ + "SYNTHETIC-PAPER-1-EVIDENCE", + "SYNTHETIC-PAPER-2-EVIDENCE", + ] + for value in expected_evidence: + require(value in evidence_text, f"Lectern evidence omitted {value!r}") + + llm = FakeLLM(scripted_deck(meta)) + state = GenerationState( + job_id="synthetic-golden-path", + evidence=ingested.assets, + tables=ingested.tables, + max_retries=0, + ) + graph = build_graph( + llm, + out_dir=output_dir, + planner=build_outline, + ) + config = {"configurable": {"thread_id": state.job_id}} + graph.invoke(state.model_dump(), config) + snapshot = graph.get_state(config) + + require( + "approval" in snapshot.next, + "Production graph did not pause at the outline approval gate", + ) + require( + not field(snapshot.values, "user_approved_outline", False), + "Production graph bypassed human approval", + ) + require( + field(snapshot.values, "output_path") is None, + "Production graph assigned an output before approval", + ) + require( + not list(output_dir.rglob("*.pptx")), + "A persistent PPTX was created before outline approval", + ) + require(len(llm.calls) == 1, "FakeLLM was not called exactly once") + prompt = llm.calls[0]["prompt"] + for value in expected_evidence: + require(value in prompt, f"Planner prompt omitted {value!r}") + + final = graph.invoke(Command(resume={"approved": True}), config) + require(field(final, "user_approved_outline") is True, "Approval was not recorded") + approved_value = field(final, "output_path") + require(isinstance(approved_value, str), "Approval did not produce an output path") + approved_path = Path(approved_value) + require(approved_path.is_file(), "Approved deck was not generated") + + presentation = Presentation(str(approved_path)) + require(len(presentation.slides) == 2, "Generated deck has the wrong slide count") + table_shapes = [ + shape + for slide in presentation.slides + for shape in slide.shapes + if getattr(shape, "has_table", False) + ] + require(len(table_shapes) == 1, "Generated deck lacks one native table") + table = table_shapes[0].table + require(len(table.rows) == 3, "Native table lost a synthetic evidence row") + require(len(table.columns) == 3, "Native table lost a column") + + title_shape = next( + ( + shape + for slide in presentation.slides + for shape in slide.shapes + if getattr(shape, "has_text_frame", False) + and "[SYNTHETIC] Dual-paper evidence review" in shape.text + ), + None, + ) + require(title_shape is not None, "Generated deck lacks editable title text") + title_shape.text_frame.text = "[SYNTHETIC] Edited evidence review" + table.cell(1, 1).text = "SYNTHETIC-EDIT-CONFIRMED" + + edited_path = output_dir / "edited-synthetic-deck.pptx" + presentation.save(str(edited_path)) + reopened = Presentation(str(edited_path)) + reopened_text = all_text(reopened) + require( + "[SYNTHETIC] Edited evidence review" in reopened_text, + "Edited title did not survive reopening", + ) + require( + "SYNTHETIC-EDIT-CONFIRMED" in reopened_text, + "Edited table cell did not survive reopening", + ) + + return { + "status": "passed", + "handoff_schema": meta["schema_version"], + "paper_count": len(papers), + "evidence_asset_count": len(ingested.assets), + "warning_count": len(ingested.warnings), + "production_graph_exercised": True, + "planner_used_source_evidence": True, + "hard_stop_blocked_pptx": True, + "approved_slide_count": len(reopened.slides), + "native_table_count": len(table_shapes), + "editable_after_reopen": True, + } + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--handoff", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + report = run(args.handoff.resolve(), args.output_dir.resolve()) + except (OSError, ValueError, WorkerFailure) as exc: + print(f"FAIL: {exc}", file=sys.stderr) + return 1 + print(json.dumps(report, ensure_ascii=True, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/e2e_general_intake.py b/tests/e2e_general_intake.py new file mode 100644 index 0000000..38577a2 --- /dev/null +++ b/tests/e2e_general_intake.py @@ -0,0 +1,253 @@ +"""Credential-free general-project intake acceptance path. + +This script exercises installed public commands against synthetic files only. It +proves that an engineering repository is registered rather than copied, while +selected Markdown/PDF documents can be previewed, migrated, verified, and rolled +back without exposing local paths in public reports. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + + +class E2EFailure(RuntimeError): + """The installed public workflow did not satisfy its release contract.""" + + +def require(condition: bool, message: str) -> None: + if not condition: + raise E2EFailure(message) + + +def run_cli( + executable: str, + arguments: list[str], + *, + environment: dict[str, str], +) -> tuple[str, dict[str, object]]: + completed = subprocess.run( + [executable, *arguments, "--json"], + check=False, + capture_output=True, + text=True, + encoding="utf-8", + errors="strict", + env=environment, + shell=False, + timeout=30, + ) + require(completed.returncode == 0, "installed command returned a failure") + require(not completed.stderr.strip(), "installed command wrote to stderr") + try: + report = json.loads(completed.stdout) + except json.JSONDecodeError as exc: + raise E2EFailure("installed command did not return one JSON report") from exc + require(isinstance(report, dict), "installed command returned a non-object report") + return completed.stdout, report + + +def require_private_values_absent(payloads: list[str], values: list[str]) -> None: + combined = "\n".join(payloads) + for index, value in enumerate(values): + variants = {value, value.replace("\\", "\\\\"), value.replace("\\", "/")} + require( + all(variant not in combined for variant in variants if variant), + f"a content-free report exposed private synthetic value {index}", + ) + + +def find_scriptorium() -> str | None: + configured = os.environ.get("SCRIPTORIUM_E2E_EXECUTABLE") + if configured: + candidate = Path(configured) + return os.fspath(candidate) if candidate.is_file() else None + discovered = shutil.which("scriptorium") + if discovered: + return discovered + executable = Path(sys.executable) + names = ("scriptorium.exe", "scriptorium") + for directory in (executable.parent, executable.parent / "Scripts"): + for name in names: + candidate = directory / name + if candidate.is_file(): + return os.fspath(candidate) + return None + + +def main() -> int: + executable = find_scriptorium() + require(executable is not None, "installed scriptorium command was not found") + + with tempfile.TemporaryDirectory(prefix="scriptorium-general-intake-") as temporary: + root = Path(temporary) + workspace = root / "workspace" + provenance_home = root / "memory" + linked_repo = root / "linked-system" + sources = root / "selected-docs" + config_dir = root / "config" + linked_repo.mkdir() + sources.mkdir() + (linked_repo / "service.py").write_text( + 'print("synthetic")\n', encoding="utf-8" + ) + (linked_repo / ".env").write_text( + "SYNTHETIC_ONLY=true\n", encoding="utf-8" + ) + (sources / "maintenance.md").write_text( + "# Synthetic maintenance notes\n", encoding="utf-8" + ) + (sources / "manual.pdf").write_bytes(b"%PDF-1.4\nsynthetic\n") + + environment = os.environ.copy() + environment.update( + { + "APPDATA": os.fspath(root / "appdata"), + "LOCALAPPDATA": os.fspath(root / "localappdata"), + "XDG_STATE_HOME": os.fspath(root / "state"), + "PYTHONDONTWRITEBYTECODE": "1", + } + ) + for directory in (root / "appdata", root / "localappdata", root / "state"): + directory.mkdir() + + init_arguments = [ + "init", + "--workspace", + os.fspath(workspace), + "--provenance-home", + os.fspath(provenance_home), + "--project-id", + "synthetic-system", + "--title", + "Synthetic system maintenance", + "--template", + "engineering", + "--linked-repo", + os.fspath(linked_repo), + "--host", + "codex", + "--context", + "Trace a synthetic maintenance issue.", + "--config-dir", + os.fspath(config_dir), + ] + content_free_outputs: list[str] = [] + raw, preview = run_cli(executable, init_arguments, environment=environment) + require(preview.get("status") == "planned", "init preview was not planned") + require(not workspace.exists(), "init preview wrote the workspace") + + raw, initialized = run_cli( + executable, [*init_arguments, "--run"], environment=environment + ) + require( + initialized.get("status") == "initialized", "engineering init did not run" + ) + project_note = workspace / "Projects" / "synthetic-system.md" + project_text = project_note.read_text(encoding="utf-8") + require('profile: "engineering"' in project_text, "engineering profile was lost") + require( + os.fspath(linked_repo).replace("\\", "/") in project_text, + "linked repository was not registered", + ) + require( + not any(path.name in {"service.py", ".env"} for path in workspace.rglob("*")), + "linking a repository copied its files into the workspace", + ) + + raw, inventory = run_cli( + executable, + ["inventory", "--source", os.fspath(sources)], + environment=environment, + ) + content_free_outputs.append(raw) + summary = inventory.get("summary") + require(isinstance(summary, dict), "inventory summary was missing") + require(summary.get("candidates") == 2, "inventory did not find two documents") + require(summary.get("markdown") == 1, "inventory lost the Markdown document") + require(summary.get("pdf") == 1, "inventory lost the PDF document") + + migration_identity = [ + "--workspace", + os.fspath(workspace), + "--batch-id", + "selected-docs-001", + ] + raw, plan = run_cli( + executable, + ["migrate", "plan", "--source", os.fspath(sources), *migration_identity], + environment=environment, + ) + content_free_outputs.append(raw) + require(plan.get("status") == "planned", "migration preview was not planned") + require(not (workspace / "Sources").exists(), "migration preview wrote files") + + raw, applied = run_cli( + executable, + ["migrate", "apply", "--source", os.fspath(sources), *migration_identity], + environment=environment, + ) + content_free_outputs.append(raw) + applied_summary = applied.get("summary") + require(isinstance(applied_summary, dict), "migration summary was missing") + require(applied_summary.get("files") == 2, "migration did not copy two documents") + + raw, verified = run_cli( + executable, + ["migrate", "verify", *migration_identity], + environment=environment, + ) + content_free_outputs.append(raw) + require(verified.get("status") == "applied", "migration verification failed") + + raw, rolled_back = run_cli( + executable, + ["migrate", "rollback", *migration_identity], + environment=environment, + ) + content_free_outputs.append(raw) + require(rolled_back.get("status") == "rolled-back", "rollback did not finish") + imported = workspace / "Sources" / "Imported" / "selected-docs-001" + require( + not any(path.is_file() for path in imported.rglob("*")), + "rollback left an owned imported file", + ) + require( + (sources / "maintenance.md").read_text(encoding="utf-8") + == "# Synthetic maintenance notes\n", + "migration changed its source", + ) + + require_private_values_absent( + content_free_outputs, + [ + os.fspath(root), + os.fspath(workspace), + os.fspath(linked_repo), + "service.py", + ".env", + "maintenance.md", + "manual.pdf", + "SYNTHETIC_ONLY=true", + ], + ) + + print( + "PASS: engineering init, linked-repository registration, explicit document " + "inventory, migration, verification, privacy, and rollback" + ) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except E2EFailure as exc: + print(f"FAIL: {exc}", file=sys.stderr) + raise SystemExit(1) from exc diff --git a/tests/e2e_install_lifecycle.py b/tests/e2e_install_lifecycle.py new file mode 100644 index 0000000..e73681e --- /dev/null +++ b/tests/e2e_install_lifecycle.py @@ -0,0 +1,840 @@ +#!/usr/bin/env python3 +"""Offline clean-install lifecycle acceptance for the Scriptorium runtime. + +Only explicitly supplied local source checkouts are used. The runner stages the +public package trees into a temporary directory, builds local wheels without an +index or build isolation, then verifies install, uninstall, reinstall, doctor, +standalone component operation, and the credential-free demo in a brand-new +virtual environment. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import tomllib +import venv +from pathlib import Path +from typing import Any + + +PACKAGES = { + "scriptorium-suite": Path("src") / "scriptorium", + "scriptorium-steward": Path("src") / "steward", + "provenance": Path("provenance"), +} +MODULES = { + "scriptorium-suite": "scriptorium", + "scriptorium-steward": "steward", + "provenance": "provenance", +} +ENTRIES = ("scriptorium", "steward", "prov-status", "prov-sync-pull") +SYSTEM_ENV = { + "COMSPEC", + "LANG", + "LC_ALL", + "LC_CTYPE", + "NUMBER_OF_PROCESSORS", + "PATH", + "PATHEXT", + "PROCESSOR_ARCHITECTURE", + "SYSTEMDRIVE", + "SYSTEMROOT", + "TERM", + "WINDIR", +} +RELEASE = re.compile(r"^(\d+)\.(\d+)\.(\d+)(?:[.+-][A-Za-z0-9.-]+)?$") + + +class LifecycleFailure(RuntimeError): + def __init__(self, stage: str, message: str): + super().__init__(message) + self.stage = stage + self.safe_message = message + + +def project_identity(root: Path, expected_name: str) -> dict[str, str]: + try: + resolved = root.resolve(strict=True) + except OSError as exc: + raise LifecycleFailure("validate-sources", "A required source is unavailable.") from exc + if root.is_symlink() or not resolved.is_dir(): + raise LifecycleFailure("validate-sources", "A source root is unsafe.") + pyproject = resolved / "pyproject.toml" + if pyproject.is_symlink() or not pyproject.is_file(): + raise LifecycleFailure("validate-sources", "A source lacks pyproject.toml.") + try: + project = tomllib.loads(pyproject.read_text(encoding="utf-8"))["project"] + name, version = project["name"], project["version"] + except (OSError, KeyError, TypeError, tomllib.TOMLDecodeError) as exc: + raise LifecycleFailure("validate-sources", "A package identity is invalid.") from exc + if name != expected_name or not isinstance(version, str) or not version: + raise LifecycleFailure("validate-sources", "A package identity does not match.") + return {"name": name, "version": version} + + +def release_relation(previous: str, current: str) -> str: + old, new = RELEASE.fullmatch(previous), RELEASE.fullmatch(current) + if old is None or new is None: + return "unsupported" + old_core = tuple(int(value) for value in old.groups()) + new_core = tuple(int(value) for value in new.groups()) + if old_core < new_core: + return "previous-is-older" + if old_core == new_core: + return "same" + return "previous-is-newer" + + +def isolated_environment(root: Path, scripts: Path | None = None) -> dict[str, str]: + profile = root / "profile" + runtime_temp = root / "runtime-temp" + appdata = profile / "AppData" / "Roaming" + localappdata = profile / "AppData" / "Local" + for directory in (profile, runtime_temp, appdata, localappdata, root / "pip-cache"): + directory.mkdir(parents=True, exist_ok=True) + env = { + name: value + for name, value in os.environ.items() + if name.upper() in SYSTEM_ENV + } + path = [str(scripts)] if scripts is not None else [] + if env.get("PATH"): + path.append(env["PATH"]) + env.update( + { + "HOME": str(profile), + "USERPROFILE": str(profile), + "APPDATA": str(appdata), + "LOCALAPPDATA": str(localappdata), + "TEMP": str(runtime_temp), + "TMP": str(runtime_temp), + "TMPDIR": str(runtime_temp), + "CODEX_HOME": str(profile / ".codex"), + "SCRIPTORIUM_CONFIG_DIR": str(root / "config"), + "PIP_CACHE_DIR": str(root / "pip-cache"), + "PIP_CONFIG_FILE": os.devnull, + "PIP_DISABLE_PIP_VERSION_CHECK": "1", + "PIP_NO_INDEX": "1", + "PIP_NO_INPUT": "1", + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONIOENCODING": "utf-8", + "PYTHONNOUSERSITE": "1", + "PYTHONUTF8": "1", + "PATH": os.pathsep.join(path), + } + ) + return env + + +def run( + command: list[str | Path], + *, + stage: str, + cwd: Path, + env: dict[str, str], + timeout: int = 180, +) -> subprocess.CompletedProcess[str]: + try: + result = subprocess.run( + [str(value) for value in command], + cwd=cwd, + env=env, + capture_output=True, + text=True, + encoding="utf-8", + errors="strict", + shell=False, + timeout=timeout, + check=False, + ) + except (OSError, subprocess.TimeoutExpired, UnicodeError) as exc: + raise LifecycleFailure(stage, "A local subprocess could not complete.") from exc + if result.returncode != 0: + raise LifecycleFailure(stage, "A local subprocess returned a failure.") + return result + + +def copy_public_tree(source: Path, destination: Path) -> None: + if source.is_symlink() or not source.is_dir(): + raise LifecycleFailure("stage-sources", "A public source tree is unsafe.") + destination.mkdir(parents=True, exist_ok=False) + for item in sorted(source.rglob("*"), key=lambda path: path.as_posix()): + relative = item.relative_to(source) + if "__pycache__" in relative.parts or item.suffix in {".pyc", ".pyo"}: + continue + if item.is_symlink(): + raise LifecycleFailure("stage-sources", "A public source contains a link.") + target = destination / relative + if item.is_dir(): + target.mkdir(parents=True, exist_ok=True) + elif item.is_file(): + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(item, target) + else: + raise LifecycleFailure("stage-sources", "A public source contains a special file.") + + +def copy_metadata(source: Path, destination: Path) -> None: + for filename in ("pyproject.toml", "README.md", "LICENSE", "NOTICE"): + candidate = source / filename + if not candidate.exists(): + continue + if candidate.is_symlink() or not candidate.is_file(): + raise LifecycleFailure("stage-sources", "A metadata file is unsafe.") + shutil.copy2(candidate, destination / filename) + + +def stage_package(source: Path, destination: Path, distribution: str) -> Path: + package = PACKAGES.get(distribution) + if package is None or destination.exists() or destination.is_symlink(): + raise LifecycleFailure("stage-sources", "A package staging request is invalid.") + destination.mkdir(parents=True) + copy_metadata(source, destination) + if not (destination / "pyproject.toml").is_file(): + raise LifecycleFailure("stage-sources", "Staged metadata is incomplete.") + copy_public_tree(source / package, destination / package) + return destination + + +def stage_spec(source: Path, destination: Path) -> Path: + project_identity(source, "scriptorium-spec") + if destination.exists() or destination.is_symlink(): + raise LifecycleFailure("stage-sources", "The Spec staging target is occupied.") + destination.mkdir(parents=True) + copy_metadata(source, destination) + for directory in ("tools", "examples", "schemas"): + copy_public_tree(source / directory, destination / directory) + return destination + + +def build_wheel( + source: Path, + *, + distribution: str, + label: str, + root: Path, + env: dict[str, str], +) -> tuple[Path, Path]: + staged = stage_package( + source, + root / "staged-sources" / label, + distribution, + ) + wheel_dir = root / "wheels" / label + wheel_dir.mkdir(parents=True) + run( + [ + sys.executable, + "-m", + "pip", + "wheel", + "--no-index", + "--no-deps", + "--no-build-isolation", + "--wheel-dir", + wheel_dir, + staged, + ], + stage=f"build-{label}", + cwd=root, + env=env, + ) + wheels = list(wheel_dir.glob("*.whl")) + if len(wheels) != 1 or wheels[0].is_symlink(): + raise LifecycleFailure(f"build-{label}", "The local wheel build was ambiguous.") + return wheels[0], staged + + +def venv_paths(root: Path) -> tuple[Path, Path]: + scripts = root / ("Scripts" if os.name == "nt" else "bin") + return scripts / ("python.exe" if os.name == "nt" else "python"), scripts + + +def entry(scripts: Path, name: str) -> Path: + return scripts / (f"{name}.exe" if os.name == "nt" else name) + + +def install( + python: Path, + wheels: list[Path], + *, + cwd: Path, + env: dict[str, str], + stage: str, + force: bool = False, +) -> None: + command: list[str | Path] = [ + python, + "-m", + "pip", + "install", + "--no-index", + "--no-deps", + ] + if force: + command.append("--force-reinstall") + run([*command, *wheels], stage=stage, cwd=cwd, env=env) + + +def package_state( + python: Path, *, cwd: Path, env: dict[str, str], stage: str +) -> dict[str, dict[str, Any]]: + code = ( + "import importlib.metadata as m, importlib.util, json\n" + f"items={tuple(MODULES.items())!r}\n" + "result={}\n" + "for dist,module in items:\n" + " try: version=m.version(dist)\n" + " except m.PackageNotFoundError: version=None\n" + " result[dist]={'version':version,'importable':importlib.util.find_spec(module) is not None}\n" + "print(json.dumps(result,sort_keys=True))\n" + ) + output = run( + [python, "-I", "-c", code], + stage=stage, + cwd=cwd, + env=env, + ).stdout + try: + value = json.loads(output) + except json.JSONDecodeError as exc: + raise LifecycleFailure(stage, "Package verification did not return JSON.") from exc + if not isinstance(value, dict): + raise LifecycleFailure(stage, "Package verification returned an invalid value.") + return value + + +def verify_installed( + python: Path, + scripts: Path, + expected: dict[str, str], + *, + cwd: Path, + env: dict[str, str], + stage: str, +) -> None: + state = package_state(python, cwd=cwd, env=env, stage=stage) + for distribution, version in expected.items(): + item = state.get(distribution) + if ( + not isinstance(item, dict) + or item.get("version") != version + or item.get("importable") is not True + ): + raise LifecycleFailure(stage, "An installed package failed verification.") + for name in ENTRIES: + candidate = entry(scripts, name) + if candidate.is_symlink() or not candidate.is_file(): + raise LifecycleFailure(stage, "A required console entry is unavailable.") + if run( + [entry(scripts, "scriptorium"), "--version"], + stage=stage, + cwd=cwd, + env=env, + ).stdout.strip() != f"scriptorium {expected['scriptorium-suite']}": + raise LifecycleFailure(stage, "The Scriptorium version did not match.") + if run( + [entry(scripts, "steward"), "--version"], + stage=stage, + cwd=cwd, + env=env, + ).stdout.strip() != f"steward {expected['scriptorium-steward']}": + raise LifecycleFailure(stage, "The Steward version did not match.") + try: + capabilities = json.loads( + run( + [entry(scripts, "prov-sync-pull"), "--capabilities", "--json"], + stage=stage, + cwd=cwd, + env=env, + ).stdout + ) + except json.JSONDecodeError as exc: + raise LifecycleFailure(stage, "The Provenance capability probe was invalid.") from exc + if ( + not isinstance(capabilities, dict) + or capabilities.get("operation") != "pull.capabilities" + or capabilities.get("generated_by", {}).get("version") != expected["provenance"] + ): + raise LifecycleFailure(stage, "The Provenance version did not match.") + + +def tree_snapshot(root: Path, *, stage: str) -> tuple[tuple[str, str, int, int], ...]: + try: + resolved = root.resolve(strict=True) + except OSError as exc: + raise LifecycleFailure(stage, "A standalone data root is unavailable.") from exc + if root.is_symlink() or not resolved.is_dir(): + raise LifecycleFailure(stage, "A standalone data root is unsafe.") + try: + records = [ + (".", "directory", 0, resolved.stat().st_mtime_ns), + ] + for path in sorted(resolved.rglob("*"), key=lambda item: item.as_posix()): + if path.is_symlink(): + raise LifecycleFailure(stage, "A standalone data root contains a link.") + relative = path.relative_to(resolved).as_posix() + stat = path.stat() + if path.is_dir(): + kind, size = "directory", 0 + elif path.is_file(): + kind, size = "file", stat.st_size + else: + raise LifecycleFailure( + stage, "A standalone data root contains a special file." + ) + records.append((relative, kind, size, stat.st_mtime_ns)) + except OSError as exc: + raise LifecycleFailure(stage, "A standalone data root could not be inspected.") from exc + return tuple(records) + + +def verify_standalone_components( + scripts: Path, + *, + root: Path, + env: dict[str, str], +) -> None: + provenance_home = root / "standalone-provenance" + provenance_home.mkdir() + provenance_env = dict(env) + provenance_env["PROVENANCE_HOME"] = str(provenance_home) + before = tree_snapshot(provenance_home, stage="standalone-provenance") + run( + [entry(scripts, "prov-status")], + stage="standalone-provenance", + cwd=root, + env=provenance_env, + ) + if tree_snapshot(provenance_home, stage="standalone-provenance") != before: + raise LifecycleFailure( + "standalone-provenance", "The read-only Provenance status command wrote data." + ) + + vault = root / "standalone-steward-vault" + projects = vault / "Projects" + projects.mkdir(parents=True) + (projects / "synthetic-system.md").write_text( + "---\n" + "schema_version: project/1.0\n" + "project_id: synthetic-system\n" + "title: Synthetic system maintenance\n" + "status: active\n" + "priority: high\n" + "---\n", + encoding="utf-8", + ) + dashboard = projects / "_总纲.md" + command = [entry(scripts, "steward"), "portfolio", "--vault", vault] + run(command, stage="standalone-steward-preview", cwd=root, env=env) + if dashboard.exists(): + raise LifecycleFailure( + "standalone-steward-preview", "The Steward preview wrote a dashboard." + ) + run([*command, "--run"], stage="standalone-steward-run", cwd=root, env=env) + try: + first = dashboard.read_bytes() + text = first.decode("utf-8") + except (OSError, UnicodeError) as exc: + raise LifecycleFailure( + "standalone-steward-run", "The Steward dashboard was unavailable." + ) from exc + if ( + "<!-- steward:portfolio:begin" not in text + or "<!-- steward:portfolio:end -->" not in text + or "synthetic-system" not in text + ): + raise LifecycleFailure( + "standalone-steward-run", "The Steward dashboard was incomplete." + ) + run([*command, "--run"], stage="standalone-steward-idempotent", cwd=root, env=env) + try: + second = dashboard.read_bytes() + except OSError as exc: + raise LifecycleFailure( + "standalone-steward-idempotent", "The Steward dashboard was unavailable." + ) from exc + if second != first: + raise LifecycleFailure( + "standalone-steward-idempotent", "The Steward dashboard was not idempotent." + ) + + +def uninstall( + python: Path, scripts: Path, *, cwd: Path, env: dict[str, str] +) -> None: + run( + [python, "-m", "pip", "uninstall", "--yes", *PACKAGES], + stage="uninstall", + cwd=cwd, + env=env, + ) + state = package_state(python, cwd=cwd, env=env, stage="verify-uninstalled") + if any( + not isinstance(item, dict) + or item.get("version") is not None + or item.get("importable") is not False + for item in state.values() + ): + raise LifecycleFailure("verify-uninstalled", "A package remained after uninstall.") + if any(entry(scripts, name).exists() for name in ENTRIES): + raise LifecycleFailure("verify-uninstalled", "A console entry remained after uninstall.") + + +def all_strings(value: Any) -> list[str]: + if isinstance(value, str): + return [value] + if isinstance(value, dict): + return [ + text + for key, item in value.items() + for text in (*all_strings(key), *all_strings(item)) + ] + if isinstance(value, list): + return [text for item in value for text in all_strings(item)] + return [] + + +def verify_demo( + scripts: Path, + *, + root: Path, + spec: Path, + steward: Path, + provenance: Path, + env: dict[str, str], +) -> None: + command = entry(scripts, "scriptorium") + common = [ + "--spec-root", + spec, + "--steward-root", + steward, + "--provenance-root", + provenance, + ] + run( + [command, "doctor", "--target", "demo", "--json", *common], + stage="doctor-after-reinstall", + cwd=root, + env=env, + ) + output = root / "synthetic-demo" + run( + [command, "demo", "--output", output, *common], + stage="demo-after-reinstall", + cwd=root, + env=env, + ) + try: + report = json.loads((output / "demo-report.json").read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise LifecycleFailure("demo-after-reinstall", "The demo report was invalid.") from exc + if not isinstance(report, dict) or report.get("demo_status") != "passed": + raise LifecycleFailure("demo-after-reinstall", "The synthetic demo did not pass.") + forbidden = str(root).replace("\\", "/").casefold() + if any(forbidden in text.replace("\\", "/").casefold() for text in all_strings(report)): + raise LifecycleFailure("demo-after-reinstall", "The demo report exposed its local path.") + + +def write_report(path: Path, report: dict[str, Any]) -> None: + if path.exists() or path.is_symlink(): + raise LifecycleFailure("write-report", "The report destination already exists.") + try: + parent = path.parent.resolve(strict=True) + except OSError as exc: + raise LifecycleFailure("write-report", "The report parent is unavailable.") from exc + temporary = parent / f".{path.name}.{os.getpid()}.tmp" + if temporary.exists() or temporary.is_symlink(): + raise LifecycleFailure("write-report", "The temporary report target is occupied.") + try: + with temporary.open("x", encoding="utf-8", newline="\n") as handle: + json.dump(report, handle, ensure_ascii=False, indent=2, sort_keys=True) + handle.write("\n") + os.replace(temporary, path) + except OSError as exc: + temporary.unlink(missing_ok=True) + raise LifecycleFailure("write-report", "The lifecycle report could not be written.") from exc + + +def parser() -> argparse.ArgumentParser: + value = argparse.ArgumentParser(description=__doc__) + value.add_argument("--scriptorium-root", type=Path, required=True) + value.add_argument("--spec-root", type=Path, required=True) + value.add_argument("--steward-root", type=Path, required=True) + value.add_argument("--provenance-root", type=Path, required=True) + value.add_argument("--previous-scriptorium-root", type=Path) + value.add_argument("--report", type=Path) + value.add_argument("--require-windows", action="store_true") + return value + + +def execute(args: argparse.Namespace) -> dict[str, Any]: + if args.require_windows and os.name != "nt": + raise LifecycleFailure("platform", "This acceptance run requires Windows.") + roots = { + "scriptorium-suite": args.scriptorium_root, + "scriptorium-steward": args.steward_root, + "provenance": args.provenance_root, + } + identities = { + name: project_identity(root, name) for name, root in roots.items() + } + expected = {name: value["version"] for name, value in identities.items()} + stages = [{"name": "validate-sources", "status": "passed"}] + limitations = [ + "OS-level network egress is not instrumented; pip is constrained to local sources with no index.", + "Live Codex/Claude sessions and PowerPoint editing remain manual acceptance items.", + ] + transition: dict[str, Any] = {"status": "not-requested"} + if args.previous_scriptorium_root is None: + limitations.append( + "Upgrade/downgrade is not automated because no older local source was supplied." + ) + previous = None + relation = "not-requested" + else: + previous = project_identity( + args.previous_scriptorium_root, "scriptorium-suite" + ) + relation = release_relation(previous["version"], expected["scriptorium-suite"]) + transition.update( + { + "status": "pending", + "previous_version": previous["version"], + "current_version": expected["scriptorium-suite"], + "relation": relation, + } + ) + try: + import setuptools # noqa: F401 + except ImportError as exc: + raise LifecycleFailure( + "build-tooling", "The local setuptools build backend is unavailable." + ) from exc + stages.append({"name": "build-tooling", "status": "passed"}) + + with tempfile.TemporaryDirectory(prefix="scriptorium-install-lifecycle-") as raw: + root = Path(raw) + build_env = isolated_environment(root / "build-environment") + wheels: list[Path] = [] + staged: dict[str, Path] = {} + for distribution, source in roots.items(): + wheel, staged_root = build_wheel( + source.resolve(strict=True), + distribution=distribution, + label=distribution, + root=root, + env=build_env, + ) + wheels.append(wheel) + staged[distribution] = staged_root + stages.append({"name": f"build-{distribution}", "status": "passed"}) + spec = stage_spec( + args.spec_root.resolve(strict=True), + root / "staged-sources" / "scriptorium-spec", + ) + stages.append({"name": "stage-scriptorium-spec", "status": "passed"}) + + previous_wheel = None + if previous is not None and relation == "previous-is-older": + previous_wheel, _ = build_wheel( + args.previous_scriptorium_root.resolve(strict=True), + distribution="scriptorium-suite", + label="previous-scriptorium-suite", + root=root, + env=build_env, + ) + stages.append( + {"name": "build-previous-scriptorium-suite", "status": "passed"} + ) + elif previous is not None: + transition["status"] = "not-covered" + limitations.append( + "Upgrade/downgrade is not claimed because the supplied versions are not an older-to-current pair." + ) + + environment = root / "clean-venv" + try: + venv.EnvBuilder(with_pip=True, symlinks=False).create(environment) + except OSError as exc: + raise LifecycleFailure("create-clean-venv", "The clean venv could not be created.") from exc + python, scripts = venv_paths(environment) + if not python.is_file(): + raise LifecycleFailure("create-clean-venv", "The clean venv lacks Python.") + stages.append({"name": "create-clean-venv", "status": "passed"}) + runtime_env = isolated_environment(root / "runtime-environment", scripts) + + lifecycle = { + "clean_install": "not-run", + "uninstall": "not-run", + "reinstall": "not-run", + "doctor_and_demo": "not-run", + "standalone_components": "not-run", + "version_transition": transition, + } + install(python, wheels, cwd=root, env=runtime_env, stage="clean-install") + verify_installed( + python, scripts, expected, cwd=root, env=runtime_env, stage="verify-clean-install" + ) + lifecycle["clean_install"] = "passed" + stages.extend( + [ + {"name": "clean-install", "status": "passed"}, + {"name": "verify-clean-install", "status": "passed"}, + ] + ) + uninstall(python, scripts, cwd=root, env=runtime_env) + lifecycle["uninstall"] = "passed" + stages.extend( + [ + {"name": "uninstall", "status": "passed"}, + {"name": "verify-uninstalled", "status": "passed"}, + ] + ) + install(python, wheels, cwd=root, env=runtime_env, stage="reinstall") + verify_installed( + python, scripts, expected, cwd=root, env=runtime_env, stage="verify-reinstall" + ) + lifecycle["reinstall"] = "passed" + stages.extend( + [ + {"name": "reinstall", "status": "passed"}, + {"name": "verify-reinstall", "status": "passed"}, + ] + ) + if previous_wheel is not None and previous is not None: + install( + python, + [previous_wheel], + cwd=root, + env=runtime_env, + stage="verified-downgrade", + force=True, + ) + old_expected = dict(expected) + old_expected["scriptorium-suite"] = previous["version"] + verify_installed( + python, scripts, old_expected, cwd=root, env=runtime_env, stage="verify-downgrade" + ) + install( + python, + [wheels[0]], + cwd=root, + env=runtime_env, + stage="verified-upgrade", + force=True, + ) + verify_installed( + python, scripts, expected, cwd=root, env=runtime_env, stage="verify-upgrade" + ) + transition["status"] = "passed" + stages.extend( + [ + {"name": "verified-downgrade", "status": "passed"}, + {"name": "verify-downgrade", "status": "passed"}, + {"name": "verified-upgrade", "status": "passed"}, + {"name": "verify-upgrade", "status": "passed"}, + ] + ) + verify_standalone_components( + scripts, + root=root, + env=runtime_env, + ) + lifecycle["standalone_components"] = "passed" + stages.extend( + [ + {"name": "standalone-provenance", "status": "passed"}, + {"name": "standalone-steward-preview", "status": "passed"}, + {"name": "standalone-steward-run", "status": "passed"}, + {"name": "standalone-steward-idempotent", "status": "passed"}, + ] + ) + verify_demo( + scripts, + root=root, + spec=spec, + steward=staged["scriptorium-steward"], + provenance=staged["provenance"], + env=runtime_env, + ) + lifecycle["doctor_and_demo"] = "passed" + stages.extend( + [ + {"name": "doctor-after-reinstall", "status": "passed"}, + {"name": "demo-after-reinstall", "status": "passed"}, + ] + ) + + return { + "format_version": 1, + "operation": "windows-install-lifecycle", + "status": "passed", + "components": identities, + "platform": { + "implementation": sys.implementation.name, + "python": ".".join(str(value) for value in sys.version_info[:3]), + "windows": os.name == "nt", + }, + "safety": { + "data": "synthetic-only", + "network_index": "disabled", + "build_isolation": "disabled", + "runtime_dependencies_added": False, + "temporary_workspace": True, + }, + "lifecycle": lifecycle, + "stages": stages, + "limitations": limitations, + } + + +def main(argv: list[str] | None = None) -> int: + args = parser().parse_args(argv) + try: + report = execute(args) + exit_code = 0 + except LifecycleFailure as exc: + report = { + "format_version": 1, + "operation": "windows-install-lifecycle", + "status": "failed", + "failure": {"stage": exc.stage, "message": exc.safe_message}, + } + exit_code = 1 + except Exception: + report = { + "format_version": 1, + "operation": "windows-install-lifecycle", + "status": "failed", + "failure": { + "stage": "internal", + "message": "The lifecycle runner encountered an internal failure.", + }, + } + exit_code = 2 + if args.report is not None: + try: + write_report(args.report, report) + except LifecycleFailure as exc: + report = { + "format_version": 1, + "operation": "windows-install-lifecycle", + "status": "failed", + "failure": {"stage": exc.stage, "message": exc.safe_message}, + } + exit_code = 2 + print(json.dumps(report, ensure_ascii=False, sort_keys=True)) + return exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/e2e_pull.py b/tests/e2e_pull.py index 42bfc58..14aa4cb 100644 --- a/tests/e2e_pull.py +++ b/tests/e2e_pull.py @@ -32,6 +32,7 @@ HUMAN_TEXT = "Human-owned research rationale must remain byte-identical." TRAILING_HUMAN_TEXT = "Human-owned trailing notes must also remain untouched." TIMELINE_FACT = "Captured the synthetic Codex research session through the public pull entry." +SECOND_TIMELINE_FACT = "Continued the synthetic project from the approved context capsule." CONCLUSION = "The synthetic research loop is ready for product-level validation." SYSTEM_ENV_NAMES = { "COMSPEC", @@ -141,7 +142,9 @@ def human_regions(note_bytes: bytes) -> tuple[bytes, bytes]: return body[:begin], body[end + len(END_MARKER.encode("utf-8")) :] -def isolated_environment(base: Path) -> tuple[dict[str, str], Path]: +def isolated_environment( + base: Path, *, spec_root: Path | None = None +) -> tuple[dict[str, str], Path]: profile = base / "profile" appdata = profile / "AppData" / "Roaming" localappdata = profile / "AppData" / "Local" @@ -196,6 +199,8 @@ def isolated_environment(base: Path) -> tuple[dict[str, str], Path]: } ) env["PATH"] = str(fake_bin) + os.pathsep + env.get("PATH", "") + if spec_root is not None: + env["SCRIPTORIUM_SPEC_ROOT"] = str(spec_root) return env, profile @@ -388,6 +393,15 @@ def status_arguments(*, provenance_root: Path) -> list[str]: ] +def resume_arguments(*, provenance_root: Path) -> list[str]: + return [ + "resume", + "--provenance-root", + str(provenance_root), + "--json", + ] + + def init_arguments( *, workspace: Path, @@ -398,6 +412,7 @@ def init_arguments( linked_repo: Path | None, run: bool, idea: str | None = None, + template: str = "software", ) -> list[str]: arguments = [ "init", @@ -411,6 +426,8 @@ def init_arguments( title, "--host", host, + "--template", + template, "--json", ] if linked_repo is not None: @@ -422,7 +439,13 @@ def init_arguments( return arguments -def write_rollout(profile: Path, linked_repo: Path) -> Path: +def write_rollout( + profile: Path, + linked_repo: Path, + *, + session_id: str = "e2e-codex-session", + filename: str = "rollout-e2e-pull.jsonl", +) -> Path: rollout = ( profile / ".codex" @@ -430,7 +453,7 @@ def write_rollout(profile: Path, linked_repo: Path) -> Path: / "2026" / "07" / "16" - / "rollout-e2e-pull.jsonl" + / filename ) rollout.parent.mkdir(parents=True, exist_ok=True) now = datetime.now(timezone.utc) @@ -439,7 +462,7 @@ def write_rollout(profile: Path, linked_repo: Path) -> Path: "type": "session_meta", "timestamp": now.isoformat(), "payload": { - "session_id": "e2e-codex-session", + "session_id": session_id, "cwd": linked_repo.as_posix(), }, }, @@ -448,7 +471,7 @@ def write_rollout(profile: Path, linked_repo: Path) -> Path: "timestamp": now.isoformat(), "payload": { "type": "user_message", - "message": "Validate the synthetic research capture loop.", + "message": "Validate the synthetic long-running project capture loop.", }, }, { @@ -470,21 +493,31 @@ def write_rollout(profile: Path, linked_repo: Path) -> Path: def submit_fill( - summary_id: str, *, env: dict[str, str], provenance_home: Path + summary_id: str, + *, + env: dict[str, str], + provenance_home: Path, + timeline: list[str] | None = None, + include_high_value_claims: bool = True, ) -> None: fixture = { "schema_version": "summary-fill/1.0", - "timeline": [ + "timeline": timeline or [ TIMELINE_FACT, "Confirmed that preview, execution, and approval remain separate actions.", ], - "status": "active", - "stage": "product validation", - "next_actions": ["Review the isolated E2E evidence before release."], - "conclusion": CONCLUSION, - "blocked_by": "", - "confidence": "high", } + if include_high_value_claims: + fixture.update( + { + "status": "active", + "stage": "product validation", + "next_actions": ["Review the isolated E2E evidence before release."], + "conclusion": CONCLUSION, + "blocked_by": "", + "confidence": "high", + } + ) report = invoke_provenance_json( "provenance.synclayer.pending_fill", [summary_id, "--provenance-home", str(provenance_home), "--json"], @@ -494,12 +527,14 @@ def submit_fill( require(report.get("status") == "accepted", "Public fill command refused the fill") -def run_unresolved_gate_e2e(provenance_root: Path) -> None: +def run_unresolved_gate_e2e( + provenance_root: Path, *, spec_root: Path | None = None +) -> None: with tempfile.TemporaryDirectory( prefix="scriptorium-unresolved-e2e-" ) as temporary: base = Path(temporary) - env, profile = isolated_environment(base) + env, profile = isolated_environment(base, spec_root=spec_root) workspace = base / "workspace" provenance_home = base / "provenance-home" workspace.mkdir() @@ -701,7 +736,7 @@ def run_unresolved_gate_e2e(provenance_root: Path) -> None: ) -def run_e2e(provenance_root: Path) -> None: +def run_e2e(provenance_root: Path, *, spec_root: Path | None = None) -> None: provenance_root = provenance_root.expanduser().resolve() require( (provenance_root / "pyproject.toml").is_file(), @@ -710,7 +745,7 @@ def run_e2e(provenance_root: Path) -> None: with tempfile.TemporaryDirectory(prefix="scriptorium-pull-e2e-") as temporary: base = Path(temporary) - env, profile = isolated_environment(base) + env, profile = isolated_environment(base, spec_root=spec_root) workspace = base / "workspace" provenance_home = base / "provenance-home" agent_cwd = workspace @@ -755,7 +790,7 @@ def run_e2e(provenance_root: Path) -> None: ) require( initialized.get("status") == "initialized", - "Public init did not initialize the research project", + "Public init did not initialize the software project", ) project_note = workspace / "Projects" / "alpha.md" config_path = ( @@ -816,6 +851,31 @@ def run_e2e(provenance_root: Path) -> None: env=env, ) require(host_report.get("operation") == "host.install", "Host install envelope changed") + claude_host_report = invoke_json( + [ + "host", + "install", + "claude-code", + "--json", + ], + env=env, + ) + require( + claude_host_report.get("operation") == "host.install", + "Claude Code host install envelope changed", + ) + codex_skill = workspace / ".agents" / "skills" / "scriptorium-research" / "SKILL.md" + claude_skill = workspace / ".claude" / "skills" / "scriptorium-research" / "SKILL.md" + require(codex_skill.is_file(), "Codex project adapter was not installed") + require(claude_skill.is_file(), "Claude Code project adapter was not installed") + require( + codex_skill.read_bytes() == claude_skill.read_bytes(), + "Agent adapters do not expose the same project workflow", + ) + require( + b"scriptorium resume" in codex_skill.read_bytes(), + "Installed Agent adapters do not expose project resume", + ) doctor = invoke_json( [ @@ -1004,6 +1064,55 @@ def configured_pull(run: bool) -> list[str]: require(CONCLUSION in approved_text, "Approved conclusion did not reach the progress log") require(human_regions(note_after_approval) == baseline_human, "Approval changed human-owned prose") + before_first_resume = snapshot_tree(base) + first_resume = invoke_json( + resume_arguments(provenance_root=provenance_root), + env=env, + ) + require( + snapshot_tree(base) == before_first_resume, + "Context resume changed the isolated filesystem", + ) + first_capsule = first_resume.get("capsule", {}) + first_project = first_capsule.get("project", {}) + first_progress = [ + item + for block in first_capsule.get("recent_progress", []) + if isinstance(block, dict) + for item in block.get("items", []) + ] + require( + first_project.get("status") == "active" + and first_project.get("stage") == "product validation", + "First resumed session did not receive approved project state", + ) + require( + first_project.get("conclusion") == CONCLUSION, + "First resumed session did not receive the approved conclusion", + ) + require( + TIMELINE_FACT in first_progress, + "First resumed session did not receive prior low-risk progress", + ) + require_private_values_absent( + first_resume, + label="First resume capsule", + forbidden=report_private_values, + ) + alternate_agent_resume = invoke_json( + resume_arguments(provenance_root=provenance_root), + env=env, + ) + require( + alternate_agent_resume.get("capsule") == first_resume.get("capsule"), + "Fresh Agent processes did not recover the same project context", + ) + require_private_values_absent( + alternate_agent_resume, + label="Alternate Agent resume capsule", + forbidden=report_private_values, + ) + approvals_after_approval = approvals_path.read_bytes() final = invoke_json( configured_pull(run=True), @@ -1018,6 +1127,120 @@ def configured_pull(run: bool) -> list[str]: require(approvals_path.read_bytes() == approvals_after_approval, "Final rerun changed Approvals.md") require(human_regions(project_note.read_bytes()) == baseline_human, "Final rerun changed human-owned prose") + second_session_id = "e2e-codex-session-two" + second_rollout = write_rollout( + profile, + agent_cwd, + session_id=second_session_id, + filename="rollout-e2e-pull-two.jsonl", + ) + second_private_values = [ + *report_private_values, + second_session_id, + second_rollout, + ] + second_run = invoke_json( + configured_pull(run=True), + env=env, + ) + require( + second_run.get("summary", {}).get("codex_enqueued") == 1, + "Second session was not enqueued exactly once", + ) + require( + second_run.get("summary", {}).get("scaffolded") == 1, + "Second session did not create one pending scaffold", + ) + require_private_values_absent( + second_run, + label="Second-session report", + forbidden=second_private_values, + ) + + second_scaffolds = read_pending_scaffolds( + env=env, provenance_home=provenance_home + ) + require( + len(second_scaffolds) == 1, + "Second session did not expose exactly one pending scaffold", + ) + second_summary_id = second_scaffolds[0].get("summary_id") + require( + isinstance(second_summary_id, str) and second_summary_id != summary_id, + "Second session did not receive a distinct stable summary id", + ) + submit_fill( + second_summary_id, + env=env, + provenance_home=provenance_home, + timeline=[SECOND_TIMELINE_FACT], + include_high_value_claims=False, + ) + second_applied = invoke_json( + configured_pull(run=True), + env=env, + ) + require( + second_applied.get("summary", {}).get("applied") == 1, + "Second session timeline was not applied", + ) + require( + second_applied.get("summary", {}).get("pending_approval") == 0, + "Timeline-only second session created a high-value approval", + ) + require_private_values_absent( + second_applied, + label="Second-session apply report", + forbidden=second_private_values, + ) + note_after_second = project_note.read_bytes() + require( + note_after_second.decode("utf-8").count(SECOND_TIMELINE_FACT) == 1, + "Second session timeline was not appended exactly once", + ) + require( + human_regions(note_after_second) == baseline_human, + "Second session changed human-owned prose", + ) + + before_second_resume = snapshot_tree(base) + second_resume = invoke_json( + resume_arguments(provenance_root=provenance_root), + env=env, + ) + require( + snapshot_tree(base) == before_second_resume, + "Second context resume changed the isolated filesystem", + ) + second_progress = [ + item + for block in second_resume.get("capsule", {}).get("recent_progress", []) + if isinstance(block, dict) + for item in block.get("items", []) + ] + require( + TIMELINE_FACT in second_progress and SECOND_TIMELINE_FACT in second_progress, + "Second resumed session could not recover both prior session increments", + ) + require_private_values_absent( + second_resume, + label="Second resume capsule", + forbidden=second_private_values, + ) + + second_idempotent = invoke_json( + configured_pull(run=True), + env=env, + ) + require( + second_idempotent.get("summary", {}).get("applied") == 0, + "Second-session idempotency rerun re-applied a summary", + ) + require( + project_note.read_bytes() == note_after_second, + "Second-session idempotency rerun changed the project note", + ) + before_final_status = snapshot_tree(base) final_status = invoke_json( status_arguments(provenance_root=provenance_root), @@ -1051,19 +1274,27 @@ def build_parser() -> argparse.ArgumentParser: default=DEFAULT_PROVENANCE_ROOT, help="Provenance source checkout (default: sibling ../Provenance)", ) + parser.add_argument( + "--spec-root", + type=Path, + help="explicit Scriptorium Spec checkout used by child commands", + ) return parser def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) try: - run_unresolved_gate_e2e(args.provenance_root.expanduser().resolve()) - run_e2e(args.provenance_root) + spec_root = args.spec_root.expanduser().resolve() if args.spec_root else None + run_unresolved_gate_e2e( + args.provenance_root.expanduser().resolve(), spec_root=spec_root + ) + run_e2e(args.provenance_root, spec_root=spec_root) except E2EFailure as exc: print(f"FAIL: {exc}", file=sys.stderr) return 1 print( - "PASS: init, config fallback, doctor, status, unresolved gate, pull, approval, and idempotency loop" + "PASS: software init, Codex/Claude adapters, context recovery, status, pull, approval, and idempotency loop" ) return 0 diff --git a/tests/e2e_slides.py b/tests/e2e_slides.py new file mode 100644 index 0000000..175947b --- /dev/null +++ b/tests/e2e_slides.py @@ -0,0 +1,479 @@ +#!/usr/bin/env python3 +"""Synthetic, offline Steward -> Spec -> Lectern golden-path acceptance. + +The script launches each repository through a separate Python process, creates +only synthetic fixtures under a temporary directory, and deletes them at exit. +It never reads a user library, research directory, agent profile, or credential. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +import tempfile +import zipfile +from pathlib import Path +from typing import Any, Iterable + + +SCRIPTORIUM_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_STEWARD_ROOT = SCRIPTORIUM_ROOT.parent / "steward" +DEFAULT_SPEC_ROOT = SCRIPTORIUM_ROOT.parent / "scriptorium-spec" +DEFAULT_LECTERN_ROOT = SCRIPTORIUM_ROOT.parent / "Academic-Slides-Agent" +LECTERN_WORKER = Path(__file__).with_name("_e2e_lectern_worker.py") +SYSTEM_ENV_NAMES = { + "COMSPEC", + "LANG", + "LC_ALL", + "LC_CTYPE", + "NUMBER_OF_PROCESSORS", + "PATH", + "PATHEXT", + "PROCESSOR_ARCHITECTURE", + "SYSTEMDRIVE", + "SYSTEMROOT", + "TERM", + "WINDIR", +} +SYNTHETIC_PAPERS = ( + { + "key": "SYNTHA01", + "title": "[SYNTHETIC] Offline catalyst benchmark", + "authors": ["Synthetic Researcher A"], + "year": "2025", + "doi": "10.0000/scriptorium-golden-a", + "tldr": "A synthetic benchmark records catalyst signal A.", + "abstract": "No real research data is used in this generated fixture.", + "folders": ["Synthetic/GoldenPath"], + "marker": "SYNTHETIC-PAPER-1-EVIDENCE", + }, + { + "key": "SYNTHB02", + "title": "[SYNTHETIC] Offline catalyst replication", + "authors": ["Synthetic Researcher B"], + "year": "2026", + "doi": "10.0000/scriptorium-golden-b", + "tldr": "A synthetic replication records catalyst signal B.", + "abstract": "The values and narrative are fabricated for software testing.", + "folders": ["Synthetic/GoldenPath"], + "marker": "SYNTHETIC-PAPER-2-EVIDENCE", + }, +) +EMAIL_PATTERN = re.compile( + r"\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b", re.IGNORECASE +) +SECRET_PATTERN = re.compile( + r"\b(?:sk-[A-Za-z0-9_-]{12,}|gh[pousr]_[A-Za-z0-9]{20,}|" + r"AKIA[0-9A-Z]{16})\b" +) +HOME_PATH_PATTERN = re.compile( + r"(?:[A-Z]:[\\/](?:Users|Documents and Settings)[\\/][^<>\"'\s]+|" + r"/(?:home|Users)/[^<>\"'\s]+)", + re.IGNORECASE, +) + + +class E2EFailure(RuntimeError): + """A cross-repository acceptance invariant failed.""" + + +def require(condition: bool, message: str) -> None: + if not condition: + raise E2EFailure(message) + + +def _pdf_escape(value: str) -> str: + return value.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)") + + +def build_pdf(lines: Iterable[str]) -> bytes: + """Build a deterministic, one-page PDF with extractable Helvetica text.""" + rendered_lines = list(lines) + require(bool(rendered_lines), "A synthetic PDF needs at least one text line") + commands = ["BT", "/F1 10 Tf", "72 740 Td", "12 TL"] + for index, line in enumerate(rendered_lines): + if index: + commands.append("T*") + commands.append(f"({_pdf_escape(line)}) Tj") + commands.append("ET") + stream = ("\n".join(commands) + "\n").encode("ascii") + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + ( + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + b"/Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>" + ), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + ( + f"<< /Length {len(stream)} >>\nstream\n".encode("ascii") + + stream + + b"endstream" + ), + ] + + output = bytearray(b"%PDF-1.4\n%\xe2\xe3\xcf\xd3\n") + offsets = [0] + for number, body in enumerate(objects, start=1): + offsets.append(len(output)) + output.extend(f"{number} 0 obj\n".encode("ascii")) + output.extend(body) + output.extend(b"\nendobj\n") + xref_offset = len(output) + output.extend(f"xref\n0 {len(objects) + 1}\n".encode("ascii")) + output.extend(b"0000000000 65535 f \n") + for offset in offsets[1:]: + output.extend(f"{offset:010d} 00000 n \n".encode("ascii")) + output.extend( + ( + f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\n" + f"startxref\n{xref_offset}\n%%EOF\n" + ).encode("ascii") + ) + return bytes(output) + + +def synthetic_pdf_lines(paper: dict[str, Any]) -> list[str]: + base = [ + str(paper["marker"]), + str(paper["title"]), + f"DOI {paper['doi']}", + "This document is fully synthetic and exists only for offline acceptance.", + "No private library, conversation, account, experiment, or credential is used.", + ] + base.extend( + f"Synthetic observation {index:02d}: catalyst signal remains reproducible " + "under a fabricated control condition." + for index in range(1, 14) + ) + return base + + +def isolated_environment(base: Path) -> dict[str, str]: + profile = base / "profile" + runtime_temp = base / "runtime-temp" + directories = { + "HOME": profile, + "USERPROFILE": profile, + "APPDATA": profile / "AppData" / "Roaming", + "LOCALAPPDATA": profile / "AppData" / "Local", + "TEMP": runtime_temp, + "TMP": runtime_temp, + "TMPDIR": runtime_temp, + "XDG_CACHE_HOME": profile / ".cache", + } + for directory in directories.values(): + directory.mkdir(parents=True, exist_ok=True) + env = { + name: os.environ[name] + for name in SYSTEM_ENV_NAMES + if name in os.environ + } + env.update({name: str(path) for name, path in directories.items()}) + env.update( + { + "ASA_PDF_PARSER": "pdfplumber", + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONIOENCODING": "utf-8", + "PYTHONNOUSERSITE": "1", + "PYTHONUTF8": "1", + } + ) + return env + + +def component_python(root: Path, *, require_venv: bool = False) -> Path: + candidates = ( + root / ".venv" / "Scripts" / "python.exe", + root / ".venv" / "bin" / "python", + ) + for candidate in candidates: + if candidate.is_file(): + return candidate + if require_venv: + raise E2EFailure( + "Lectern has no .venv; run `uv sync --all-packages --locked --no-dev`" + ) + return Path(sys.executable) + + +def run_command( + command: list[str], + *, + cwd: Path, + env: dict[str, str], + label: str, + timeout: int = 180, +) -> subprocess.CompletedProcess[str]: + try: + completed = subprocess.run( + command, + cwd=cwd, + env=env, + capture_output=True, + text=True, + encoding="utf-8", + errors="strict", + shell=False, + timeout=timeout, + check=False, + ) + except (OSError, subprocess.TimeoutExpired, UnicodeError) as exc: + raise E2EFailure(f"{label} could not execute") from exc + if completed.returncode != 0: + raise E2EFailure( + f"{label} failed with exit {completed.returncode}: " + f"stdout={completed.stdout!r} stderr={completed.stderr!r}" + ) + return completed + + +def read_json(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8-sig")) + except (OSError, json.JSONDecodeError) as exc: + raise E2EFailure(f"Expected valid JSON artifact: {path.name}") from exc + require(isinstance(value, dict), f"Expected JSON object: {path.name}") + return value + + +def artifact_chunks(path: Path) -> Iterable[bytes]: + if path.suffix.lower() == ".pptx": + try: + with zipfile.ZipFile(path) as archive: + for name in sorted(archive.namelist()): + if not name.endswith("/"): + yield archive.read(name) + except (OSError, zipfile.BadZipFile) as exc: + raise E2EFailure(f"Generated PPTX is not a valid package: {path.name}") from exc + return + yield path.read_bytes() + + +def assert_public_artifacts_safe( + paths: Iterable[Path], *, forbidden_literals: Iterable[str | Path] +) -> None: + literals = [ + variant + for value in forbidden_literals + for variant in { + str(value), + str(value).replace("\\", "/"), + str(value).replace("/", "\\"), + } + if variant + ] + for path in paths: + for chunk in artifact_chunks(path): + for literal in literals: + if ( + literal.encode("utf-8", errors="ignore") in chunk + or literal.encode("utf-16le", errors="ignore") in chunk + ): + raise E2EFailure( + f"Transferable artifact exposed a local value: {path.name}" + ) + texts = ( + chunk.decode("utf-8", errors="ignore"), + chunk.decode("latin-1", errors="ignore"), + ) + for text in texts: + if "pdfPaths" in text: + raise E2EFailure( + f"Transferable artifact retained source paths: {path.name}" + ) + if EMAIL_PATTERN.search(text): + raise E2EFailure( + f"Transferable artifact contains an email: {path.name}" + ) + if SECRET_PATTERN.search(text): + raise E2EFailure( + f"Transferable artifact contains a credential pattern: {path.name}" + ) + if HOME_PATH_PATTERN.search(text): + raise E2EFailure( + f"Transferable artifact contains a home path: {path.name}" + ) + + +def create_library_kb(source_dir: Path) -> Path: + items: list[dict[str, Any]] = [] + for index, paper in enumerate(SYNTHETIC_PAPERS, start=1): + pdf_path = source_dir / f"synthetic-paper-{index}.pdf" + pdf_path.write_bytes(build_pdf(synthetic_pdf_lines(paper))) + item = { + key: value + for key, value in paper.items() + if key != "marker" + } + item["pdfPaths"] = [str(pdf_path)] + items.append(item) + kb_path = source_dir / "synthetic-library.json" + kb_path.write_text( + json.dumps( + {"schema_version": "library-kb/1.1", "items": items}, + ensure_ascii=True, + indent=2, + ), + encoding="utf-8", + ) + return kb_path + + +def validate_handoff(meta: dict[str, Any], handoff_dir: Path) -> None: + require(meta.get("schema_version") == "handoff/1.1", "Steward did not emit handoff/1.1") + require(meta.get("report_type") == "literature", "Handoff has the wrong report type") + require(meta.get("title") == "[SYNTHETIC] Offline evidence review", "Handoff title changed") + require( + isinstance(meta.get("key"), str) and len(meta["key"]) == 8, + "Handoff lacks an eight-character report key", + ) + papers = meta.get("papers") + require(isinstance(papers, list) and len(papers) == 2, "Handoff lost a paper") + for expected, actual in zip(SYNTHETIC_PAPERS, papers): + require(actual.get("title") == expected["title"], "Handoff changed a title") + require(actual.get("doi") == expected["doi"], "Handoff changed a DOI") + require("key" not in actual, "Handoff duplicated a per-paper key") + pdf_name = actual.get("pdfFilename") + require( + isinstance(pdf_name, str) and (handoff_dir / pdf_name).is_file(), + "Handoff omitted a staged PDF", + ) + + +def run_e2e(steward_root: Path, spec_root: Path, lectern_root: Path) -> dict[str, Any]: + for label, root in ( + ("Steward", steward_root), + ("Spec", spec_root), + ("Lectern", lectern_root), + ): + require(root.is_dir(), f"{label} checkout not found") + + with tempfile.TemporaryDirectory(prefix="scriptorium-slides-golden-") as raw_base: + base = Path(raw_base) + source_dir = base / "synthetic-input" + staging_dir = base / "handoff" + output_dir = base / "lectern-output" + source_dir.mkdir() + staging_dir.mkdir() + output_dir.mkdir() + env = isolated_environment(base) + kb_path = create_library_kb(source_dir) + + steward_python = component_python(steward_root) + run_command( + [ + str(steward_python), + "-m", + "steward.cli", + "pick", + *(str(paper["key"]) for paper in SYNTHETIC_PAPERS), + "--kb", + str(kb_path), + "--staging", + str(staging_dir), + "--report-type", + "literature", + "--report-title", + "[SYNTHETIC] Offline evidence review", + ], + cwd=steward_root, + env=env, + label="Steward pick", + ) + handoff_dirs = [path for path in staging_dir.iterdir() if path.is_dir()] + require(len(handoff_dirs) == 1, "Steward did not create exactly one handoff") + handoff_dir = handoff_dirs[0] + meta_path = handoff_dir / "meta.json" + meta = read_json(meta_path) + validate_handoff(meta, handoff_dir) + + spec_python = component_python(spec_root) + validated = run_command( + [str(spec_python), str(spec_root / "tools" / "validate.py"), str(meta_path)], + cwd=spec_root, + env=env, + label="Spec validation", + ) + require("ok" in validated.stdout.lower(), "Spec validator did not accept metadata") + + lectern_python = component_python(lectern_root, require_venv=True) + worker = run_command( + [ + str(lectern_python), + str(LECTERN_WORKER), + "--handoff", + str(handoff_dir), + "--output-dir", + str(output_dir), + ], + cwd=lectern_root, + env=env, + label="Lectern golden path", + ) + try: + report = json.loads(worker.stdout) + except json.JSONDecodeError as exc: + raise E2EFailure("Lectern worker did not emit one JSON object") from exc + require(isinstance(report, dict), "Lectern worker report is not an object") + require(report.get("status") == "passed", "Lectern worker did not pass") + require( + report.get("production_graph_exercised") is True, + "Lectern production-graph proof is absent", + ) + require(report.get("hard_stop_blocked_pptx") is True, "Hard-stop proof is absent") + require(report.get("editable_after_reopen") is True, "PPTX editability proof is absent") + + transferable = [ + path + for root in (handoff_dir, output_dir) + for path in root.rglob("*") + if path.is_file() + ] + generated_pptx = list(output_dir.rglob("*.pptx")) + require(len(generated_pptx) == 2, "Expected approved and edited PPTX outputs") + forbidden = [ + base, + steward_root, + spec_root, + lectern_root, + Path.home(), + os.environ.get("HOME", ""), + os.environ.get("USERPROFILE", ""), + ] + assert_public_artifacts_safe(transferable, forbidden_literals=forbidden) + return report + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--steward-root", type=Path, default=DEFAULT_STEWARD_ROOT) + parser.add_argument("--spec-root", type=Path, default=DEFAULT_SPEC_ROOT) + parser.add_argument("--lectern-root", type=Path, default=DEFAULT_LECTERN_ROOT) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + report = run_e2e( + args.steward_root.resolve(), + args.spec_root.resolve(), + args.lectern_root.resolve(), + ) + except E2EFailure as exc: + print(f"FAIL: {exc}", file=sys.stderr) + return 1 + print( + "PASS: synthetic Steward handoff, Spec validation, Lectern evidence, " + f"approval hard-stop, and editable PPTX ({report['approved_slide_count']} slides)" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_cli_init.py b/tests/test_cli_init.py index 92c1ecc..e08fb15 100644 --- a/tests/test_cli_init.py +++ b/tests/test_cli_init.py @@ -72,6 +72,11 @@ def test_preview_then_run_create_a_real_project(self): self.assertTrue( (workspace / "Projects" / "catalyst-screening.md").is_file() ) + project_note = ( + workspace / "Projects" / "catalyst-screening.md" + ).read_text(encoding="utf-8") + self.assertIn('profile: "general"', project_note) + self.assertIn("## Goal and scope", project_note) self.assertTrue(provenance_home.is_dir()) self.assertTrue(resolve_config_path(config_dir).is_file()) diff --git a/tests/test_cli_install.py b/tests/test_cli_install.py new file mode 100644 index 0000000..a11cffa --- /dev/null +++ b/tests/test_cli_install.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import contextlib +import io +import json +import tempfile +import unittest +from pathlib import Path + +from scriptorium import cli + + +class InstallCliTests(unittest.TestCase): + def invoke(self, arguments: list[str]) -> tuple[int, dict[str, object], str]: + stdout = io.StringIO() + stderr = io.StringIO() + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + code = cli.main(arguments) + return code, json.loads(stdout.getvalue()), stderr.getvalue() + + def test_core_preview_is_zero_write_json(self): + with tempfile.TemporaryDirectory() as temporary: + target = Path(temporary) / "components" + code, report, stderr = self.invoke( + ["install", "core", "--target", str(target), "--json"] + ) + self.assertEqual(code, 0) + self.assertEqual(stderr, "") + self.assertEqual(report["status"], "planned") + self.assertEqual(report["writes"], "none") + self.assertFalse(target.exists()) + + def test_capture_run_fails_without_creating_target_until_asset_exists(self): + with tempfile.TemporaryDirectory() as temporary: + target = Path(temporary) / "capture" + code, report, stderr = self.invoke( + [ + "install", + "capture", + "--target", + str(target), + "--run", + "--json", + ] + ) + self.assertEqual(code, 2) + self.assertEqual(stderr, "") + self.assertEqual(report["errors"], [{"code": "artifact_unpublished"}]) + self.assertFalse(target.exists()) + + def test_unknown_profile_is_a_stable_json_error(self): + with tempfile.TemporaryDirectory() as temporary: + target = Path(temporary) / "components" + code, report, stderr = self.invoke( + ["install", "unknown", "--target", str(target), "--json"] + ) + self.assertEqual(code, 2) + self.assertEqual(stderr, "") + self.assertEqual(report["errors"], [{"code": "unknown_profile"}]) + self.assertFalse(target.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cli_migration.py b/tests/test_cli_migration.py new file mode 100644 index 0000000..5306966 --- /dev/null +++ b/tests/test_cli_migration.py @@ -0,0 +1,370 @@ +from __future__ import annotations + +import contextlib +import io +import json +import os +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +from scriptorium import cli +from scriptorium.migration import MigrationError + + +def aggregate_report(operation: str, status: str) -> dict[str, object]: + return { + "schema_version": "migration-report/1.0", + "operation": operation, + "status": status, + "summary": { + "sources_requested": 1, + "files": 2, + "markdown": 1, + "pdf": 1, + "bytes": 24, + "changed": 0, + "unchanged": 0, + }, + "limitations": [], + } + + +class MigrationCliTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name).resolve() + self.workspace = self.root / "research workspace" + self.sources = self.root / "selected sources" + self.platform_state = self.root / "platform state" + self.workspace.mkdir() + self.sources.mkdir() + self.environment = mock.patch.dict( + os.environ, + { + "LOCALAPPDATA": str(self.platform_state), + "XDG_STATE_HOME": str(self.platform_state), + }, + clear=False, + ) + self.environment.start() + self.addCleanup(self.environment.stop) + + def invoke( + self, arguments: list[str] + ) -> tuple[int, str, str]: + stdout, stderr = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + exit_code = cli.main(arguments) + return exit_code, stdout.getvalue(), stderr.getvalue() + + def invoke_json( + self, arguments: list[str] + ) -> tuple[int, dict[str, object], str]: + exit_code, stdout, stderr = self.invoke(arguments) + return exit_code, json.loads(stdout), stderr + + def test_plan_forwards_repeated_explicit_sources(self): + first = self.sources / "first.md" + second = self.sources / "second.pdf" + expected = SimpleNamespace(report=aggregate_report("plan", "planned")) + with mock.patch( + "scriptorium.cli.plan_migration", return_value=expected + ) as plan: + exit_code, report, stderr = self.invoke_json( + [ + "migrate", + "plan", + "--source", + str(first), + "--source", + str(second), + "--workspace", + str(self.workspace), + "--batch-id", + "batch-001", + "--json", + ] + ) + + self.assertEqual(exit_code, 0) + self.assertEqual(stderr, "") + self.assertEqual(report, expected.report) + plan.assert_called_once_with( + [first, second], + workspace=self.workspace, + batch_id="batch-001", + ) + + def test_apply_uses_sources_for_first_run_and_identity_for_reapply(self): + source = self.sources / "idea.md" + planned = object() + applied = SimpleNamespace(report=aggregate_report("apply", "applied")) + with ( + mock.patch( + "scriptorium.cli.plan_migration", return_value=planned + ) as plan, + mock.patch( + "scriptorium.cli.apply_migration", return_value=applied + ) as apply, + mock.patch("scriptorium.cli.reapply_migration") as reapply, + ): + exit_code, report, _stderr = self.invoke_json( + [ + "migrate", + "apply", + "--source", + str(source), + "--workspace", + str(self.workspace), + "--batch-id", + "batch-001", + "--json", + ] + ) + self.assertEqual(exit_code, 0) + self.assertEqual(report["status"], "applied") + plan.assert_called_once_with( + [source], workspace=self.workspace, batch_id="batch-001" + ) + apply.assert_called_once_with(planned) + reapply.assert_not_called() + + unchanged = SimpleNamespace( + report=aggregate_report("apply", "unchanged") + ) + with ( + mock.patch("scriptorium.cli.plan_migration") as plan, + mock.patch("scriptorium.cli.apply_migration") as apply, + mock.patch( + "scriptorium.cli.reapply_migration", + return_value=unchanged, + ) as reapply, + ): + exit_code, report, _stderr = self.invoke_json( + [ + "migrate", + "apply", + "--workspace", + str(self.workspace), + "--batch-id", + "batch-001", + "--json", + ] + ) + self.assertEqual(exit_code, 0) + self.assertEqual(report["status"], "unchanged") + reapply.assert_called_once_with( + workspace=self.workspace, batch_id="batch-001" + ) + plan.assert_not_called() + apply.assert_not_called() + + def test_verify_and_rollback_need_only_workspace_and_batch(self): + verified = SimpleNamespace( + report=aggregate_report("verify", "applied") + ) + with mock.patch( + "scriptorium.cli.verify_migration", return_value=verified + ) as verify: + exit_code, report, _stderr = self.invoke_json( + [ + "migrate", + "verify", + "--workspace", + str(self.workspace), + "--batch-id", + "batch-001", + "--json", + ] + ) + self.assertEqual(exit_code, 0) + self.assertEqual(report["operation"], "verify") + verify.assert_called_once_with( + workspace=self.workspace, batch_id="batch-001" + ) + + loaded = object() + rolled_back = SimpleNamespace( + report=aggregate_report("rollback", "rolled-back") + ) + with ( + mock.patch( + "scriptorium.cli.load_migration", return_value=loaded + ) as load, + mock.patch( + "scriptorium.cli.rollback_migration", + return_value=rolled_back, + ) as rollback, + ): + exit_code, report, _stderr = self.invoke_json( + [ + "migrate", + "rollback", + "--workspace", + str(self.workspace), + "--batch-id", + "batch-001", + "--json", + ] + ) + self.assertEqual(exit_code, 0) + self.assertEqual(report["status"], "rolled-back") + load.assert_called_once_with( + workspace=self.workspace, batch_id="batch-001" + ) + rollback.assert_called_once_with(loaded) + + def test_real_cli_lifecycle_is_path_free_and_idempotent(self): + markdown = self.sources / "synthetic idea.md" + pdf = self.sources / "synthetic paper.pdf" + markdown.write_text("# Synthetic\n", encoding="utf-8") + pdf.write_bytes(b"%PDF-1.4\nsynthetic\n") + common = [ + "--workspace", + str(self.workspace), + "--batch-id", + "synthetic-batch", + "--json", + ] + + human_code, human_output, human_stderr = self.invoke( + [ + "migrate", + "plan", + "--source", + str(self.sources), + "--workspace", + str(self.workspace), + "--batch-id", + "synthetic-batch", + ] + ) + plan_code, planned, plan_stderr = self.invoke_json( + [ + "migrate", + "plan", + "--source", + str(self.sources), + *common, + ] + ) + apply_code, applied, apply_stderr = self.invoke_json( + [ + "migrate", + "apply", + "--source", + str(self.sources), + *common, + ] + ) + verify_code, verified, verify_stderr = self.invoke_json( + ["migrate", "verify", *common] + ) + repeat_code, repeated, repeat_stderr = self.invoke_json( + ["migrate", "apply", *common] + ) + rollback_code, rolled_back, rollback_stderr = self.invoke_json( + ["migrate", "rollback", *common] + ) + + self.assertEqual( + (plan_code, apply_code, verify_code, repeat_code, rollback_code), + (0, 0, 0, 0, 0), + ) + self.assertEqual(human_code, 0) + self.assertEqual(human_stderr, "") + self.assertIn("Migration plan: planned", human_output) + self.assertNotIn(str(self.root), human_output) + self.assertNotIn(markdown.name, human_output) + self.assertEqual( + ( + planned["status"], + applied["status"], + verified["status"], + repeated["status"], + rolled_back["status"], + ), + ("planned", "applied", "applied", "unchanged", "rolled-back"), + ) + self.assertEqual( + plan_stderr + + apply_stderr + + verify_stderr + + repeat_stderr + + rollback_stderr, + "", + ) + serialized = json.dumps( + [planned, applied, verified, repeated, rolled_back], + ensure_ascii=False, + ) + self.assertNotIn(str(self.root), serialized) + self.assertNotIn(markdown.name, serialized) + self.assertEqual(markdown.read_text(encoding="utf-8"), "# Synthetic\n") + self.assertEqual(pdf.read_bytes(), b"%PDF-1.4\nsynthetic\n") + + def test_usage_and_runtime_errors_never_echo_private_arguments(self): + sentinel = str(self.root / "private research") + arguments = [ + "migrate", + "plan", + "--source", + sentinel, + "--workspace", + sentinel, + "--batch-id", + "batch-001", + "--unknown", + sentinel, + ] + exit_code, report, stderr = self.invoke_json(arguments + ["--json"]) + self.assertEqual(exit_code, 2) + self.assertEqual(stderr, "") + self.assertEqual(report["operation"], "plan") + self.assertEqual(report["errors"], [{"code": "entry_error"}]) + self.assertNotIn(sentinel, json.dumps(report)) + + exit_code, stdout, stderr = self.invoke(arguments) + self.assertEqual(exit_code, 2) + self.assertEqual(stdout, "") + self.assertIn("invalid migrate invocation", stderr) + self.assertNotIn(sentinel, stderr) + + failures = ( + (MigrationError("target_exists"), "target_exists"), + (MigrationError(sentinel), "entry_error"), + (OSError(sentinel), "entry_error"), + ) + for failure, code in failures: + with ( + self.subTest(code=code), + mock.patch( + "scriptorium.cli.plan_migration", + side_effect=failure, + ), + ): + exit_code, report, stderr = self.invoke_json( + [ + "migrate", + "plan", + "--source", + sentinel, + "--workspace", + str(self.workspace), + "--batch-id", + "batch-001", + "--json", + ] + ) + self.assertEqual(exit_code, 2) + self.assertEqual(stderr, "") + self.assertEqual(report["errors"], [{"code": code}]) + self.assertNotIn(sentinel, json.dumps(report)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_components.py b/tests/test_components.py new file mode 100644 index 0000000..8c6afc5 --- /dev/null +++ b/tests/test_components.py @@ -0,0 +1,42 @@ +import unittest + +from scriptorium.components import ( + ComponentCatalogError, + build_component_report, + format_component_report, + load_component_catalog, +) + + +class ComponentCatalogTests(unittest.TestCase): + def test_catalog_keeps_capture_owned_by_provenance(self): + catalog = load_component_catalog() + capture = catalog.components["capture"] + self.assertEqual(capture.owner, "provenance") + self.assertEqual(capture.delivery, "release-asset") + self.assertNotIn("capture", catalog.profiles["core"]) + + def test_core_profile_selects_only_contract_and_memory_core(self): + report = build_component_report("core") + selected = { + row["id"] for row in report["components"] if row["selected"] + } + self.assertEqual(selected, {"scriptorium-spec", "provenance"}) + + def test_capture_profile_does_not_select_the_provenance_source_package(self): + report = build_component_report("capture") + selected = [row["id"] for row in report["components"] if row["selected"]] + self.assertEqual(selected, ["capture"]) + + def test_component_report_is_readable(self): + output = format_component_report(build_component_report("research")) + self.assertIn("Selected profile: research", output) + self.assertIn("* steward 0.2.0", output) + + def test_unknown_profile_fails_closed(self): + with self.assertRaisesRegex(ComponentCatalogError, "unknown"): + build_component_report("unknown") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_demo.py b/tests/test_demo.py index b1c64ea..1efde90 100644 --- a/tests/test_demo.py +++ b/tests/test_demo.py @@ -39,9 +39,9 @@ def test_compatibility_manifest_locks_alpha_baseline(self): self.assertEqual( load_compatibility(), { - "scriptorium-spec": "2.2.0", + "scriptorium-spec": "2.3.0", "steward": "0.2.0", - "provenance": "0.17.0", + "provenance": "0.18.0", }, ) @@ -54,6 +54,31 @@ def test_demo_library_fixture_is_explicitly_synthetic_and_versioned(self): self.assertEqual(data["generated_by"], "scriptorium demo fixture") self.assertEqual([item["key"] for item in data["items"]], ["DEMO0001", "DEMO0002", "DEMO0003"]) + def test_demo_research_fixtures_are_synthetic_and_share_stable_citekeys(self): + expected = { + "parsed-paper.v1.json": "parsed-paper/1.0", + "reading-note.v1.json": "reading-note/1.0", + "review.v1.json": "review/1.0", + "lineage-graph.v1.json": "lineage-graph/1.0", + } + fixtures = {} + for name, schema_version in expected.items(): + payload = resources.files("scriptorium.assets").joinpath(name).read_text( + encoding="utf-8" + ) + self.assertIn("[SYNTHETIC]", payload) + self.assertNotRegex(payload, r"(?i)(?<![A-Z0-9])[A-Z]:[\\/]") + fixtures[name] = json.loads(payload) + self.assertEqual(fixtures[name]["schema_version"], schema_version) + + self.assertEqual(fixtures["parsed-paper.v1.json"]["id"], "liu2024ActiveLearning") + self.assertEqual(fixtures["reading-note.v1.json"]["id"], "liu2024ActiveLearning") + review_rows = fixtures["review.v1.json"]["comparison_table"]["rows"] + lineage_nodes = fixtures["lineage-graph.v1.json"]["nodes"] + expected_citekeys = {"chen2022PhysicsInformed", "liu2024ActiveLearning"} + self.assertEqual({row["citekey"] for row in review_rows}, expected_citekeys) + self.assertEqual({node["citekey"] for node in lineage_nodes}, expected_citekeys) + def test_prepare_output_reuses_only_owned_directory(self): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) / "demo" diff --git a/tests/test_doctor.py b/tests/test_doctor.py index 63e9ff0..27e56ab 100644 --- a/tests/test_doctor.py +++ b/tests/test_doctor.py @@ -11,6 +11,7 @@ from scriptorium import cli from scriptorium.doctor import ( DoctorError, + PROVENANCE_BASE_COMMANDS, TARGETS, _agent_host_check, _browser_extension_check, @@ -19,6 +20,7 @@ _find_application, _host_adapter_check, _agent_capture_check, + _entry_context_check, _entry_pull_check, _project_metadata, _provenance_home_check, @@ -242,6 +244,9 @@ def test_human_output_is_stable_plain_text_with_fix(self): class DoctorProbeTests(unittest.TestCase): + def test_public_alpha_baseline_requires_research_artifact_ingest(self): + self.assertIn("prov-ingest-research", PROVENANCE_BASE_COMMANDS) + def test_invalid_lectern_environment_root_fails_closed(self): with tempfile.TemporaryDirectory() as temporary: missing = Path(temporary) / "missing-lectern" @@ -705,6 +710,52 @@ def locate(_root: Path, name: str) -> Path: self.assertIn("different Provenance environment", check["details"]["error"]) probe.assert_not_called() + def test_context_entry_requires_matching_runtime_version(self): + with mock.patch( + "scriptorium.doctor.find_script", + side_effect=lambda _root, name: Path("shared-environment") / name, + ), mock.patch( + "scriptorium.doctor._run_probe", return_value="0.17.0\n" + ) as probe: + ready = _entry_context_check( + "public-alpha", Path("Provenance"), "0.17.0" + ) + mismatch = _entry_context_check( + "public-alpha", Path("Provenance"), "0.18.0" + ) + + self.assertEqual(ready["status"], "pass") + self.assertEqual(ready["details"]["runtime_version"], "0.17.0") + self.assertEqual(mismatch["status"], "fail") + self.assertIn("runtime version", mismatch["details"]["error"]) + self.assertEqual(probe.call_args.args[0][-1], "--version") + + def test_context_entry_is_required_only_for_public_alpha(self): + public = _entry_context_check("public-alpha", None, "0.17.0") + demo = _entry_context_check("demo", None, "0.17.0") + self.assertEqual(public["status"], "fail") + self.assertEqual(demo["status"], "info") + + def test_context_command_must_share_the_provenance_environment(self): + def locate(_root: Path, name: str) -> Path: + directory = ( + "context-environment" + if name == "prov-context" + else "base-environment" + ) + return Path(directory) / name + + with mock.patch( + "scriptorium.doctor.find_script", side_effect=locate + ), mock.patch("scriptorium.doctor._run_probe") as probe: + check = _entry_context_check( + "public-alpha", Path("Provenance"), "0.17.0" + ) + + self.assertEqual(check["status"], "fail") + self.assertIn("different Provenance environment", check["details"]["error"]) + probe.assert_not_called() + def test_invalid_compatibility_shape_is_a_doctor_error(self): with mock.patch( "scriptorium.doctor.load_compatibility", side_effect=AttributeError("items") diff --git a/tests/test_e2e_slides.py b/tests/test_e2e_slides.py new file mode 100644 index 0000000..05307ca --- /dev/null +++ b/tests/test_e2e_slides.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import importlib.util +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + + +MODULE_PATH = Path(__file__).with_name("e2e_slides.py") +MODULE_SPEC = importlib.util.spec_from_file_location("e2e_slides", MODULE_PATH) +assert MODULE_SPEC and MODULE_SPEC.loader +e2e_slides = importlib.util.module_from_spec(MODULE_SPEC) +MODULE_SPEC.loader.exec_module(e2e_slides) + + +class SyntheticSlidesE2ETest(unittest.TestCase): + def test_pdf_has_valid_offsets_and_extractable_markers(self) -> None: + payload = e2e_slides.build_pdf( + ["SYNTHETIC-PAPER-1-EVIDENCE", r"escaped (value) \ test"] + ) + self.assertTrue(payload.startswith(b"%PDF-1.4")) + self.assertTrue(payload.endswith(b"%%EOF\n")) + self.assertIn(b"SYNTHETIC-PAPER-1-EVIDENCE", payload) + xref = payload.index(b"xref\n") + lines = payload[xref:].splitlines() + self.assertEqual(lines[1], b"0 6") + for object_number, row in enumerate(lines[3:8], start=1): + offset = int(row[:10]) + self.assertTrue( + payload[offset:].startswith(f"{object_number} 0 obj\n".encode("ascii")) + ) + + def test_isolated_environment_drops_credentials(self) -> None: + with tempfile.TemporaryDirectory() as raw: + with patch.dict( + os.environ, + { + "PATH": os.environ.get("PATH", ""), + "SYSTEMROOT": os.environ.get("SYSTEMROOT", ""), + "OPENAI_API_KEY": "sk-not-allowed-in-child", + "ANTHROPIC_API_KEY": "also-not-allowed", + "GITHUB_TOKEN": "ghp_not_allowed_in_child", + }, + clear=True, + ): + env = e2e_slides.isolated_environment(Path(raw)) + self.assertNotIn("OPENAI_API_KEY", env) + self.assertNotIn("ANTHROPIC_API_KEY", env) + self.assertNotIn("GITHUB_TOKEN", env) + self.assertEqual(env["ASA_PDF_PARSER"], "pdfplumber") + self.assertEqual(env["PYTHONNOUSERSITE"], "1") + + def test_privacy_scan_accepts_synthetic_payload(self) -> None: + with tempfile.TemporaryDirectory() as raw: + artifact = Path(raw) / "meta.json" + artifact.write_text( + '{"schema_version":"handoff/1.1","title":"[SYNTHETIC] Safe"}', + encoding="utf-8", + ) + e2e_slides.assert_public_artifacts_safe( + [artifact], forbidden_literals=["C:/Users/private"] + ) + + def test_privacy_scan_rejects_paths_email_and_secrets(self) -> None: + unsafe_values = ( + "C:/Users/private/research.pdf", + "researcher@example.org", + "sk-0123456789ABCDEFGHIJ", + '"pdfPaths":["synthetic.pdf"]', + ) + for index, value in enumerate(unsafe_values): + with self.subTest(value=value): + with tempfile.TemporaryDirectory() as raw: + artifact = Path(raw) / f"unsafe-{index}.txt" + artifact.write_text(value, encoding="utf-8") + with self.assertRaises(e2e_slides.E2EFailure): + e2e_slides.assert_public_artifacts_safe( + [artifact], + forbidden_literals=["C:/Users/private"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_init.py b/tests/test_init.py index d7cdcf0..0a8611a 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -1018,6 +1018,51 @@ def test_format_report_has_no_absolute_change_paths(self): f"{change['root']}:{change['path']}" for change in report["changes"] )) + def test_general_engineering_and_software_templates_are_distinct(self): + expected = { + "general": ("## Goal and scope", "## Source material"), + "engineering": ("## System context", "## Constraints and risks"), + "software": ("## Product intent", "## Design decisions"), + } + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + for template, headings in expected.items(): + arguments = self.paths(root / template) + report = run_init( + **arguments, + template=template, + idea=f"Initial context for {template}", + run=True, + ) + self.assertEqual(report["exit_code"], 0) + note = ( + arguments["workspace"] + / "Projects" + / "catalyst-screening.md" + ).read_text(encoding="utf-8") + self.assertIn("schema_version: project/1.1", note) + self.assertIn(f'profile: "{template}"', note) + self.assertIn(f"Initial context for {template}", note) + for heading in headings: + self.assertIn(heading, note) + + def test_unknown_template_is_rejected_before_writes(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + arguments = self.paths(root) + with self.assertRaisesRegex(InitError, "template"): + run_init(**arguments, template="unknown", run=True) + self.assertFalse(arguments["workspace"].exists()) + + def test_managed_roots_reject_the_user_home_directory(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + arguments = self.paths(root) + arguments["provenance_home"] = Path.home() + with self.assertRaisesRegex(InitError, "user home"): + run_init(**arguments, run=True) + self.assertFalse(arguments["workspace"].exists()) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_install_lifecycle.py b/tests/test_install_lifecycle.py new file mode 100644 index 0000000..60775ae --- /dev/null +++ b/tests/test_install_lifecycle.py @@ -0,0 +1,192 @@ +import importlib.util +import json +import os +import tempfile +import unittest +from pathlib import Path +from unittest import mock + + +SCRIPT = Path(__file__).with_name("e2e_install_lifecycle.py") +SPEC = importlib.util.spec_from_file_location("e2e_install_lifecycle", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +LIFECYCLE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(LIFECYCLE) + + +class IdentityTests(unittest.TestCase): + def test_reads_expected_identity(self): + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + (root / "pyproject.toml").write_text( + '[project]\nname="scriptorium-suite"\nversion="0.4.0"\n', + encoding="utf-8", + ) + + self.assertEqual( + LIFECYCLE.project_identity(root, "scriptorium-suite"), + {"name": "scriptorium-suite", "version": "0.4.0"}, + ) + + def test_rejects_wrong_distribution(self): + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + (root / "pyproject.toml").write_text( + '[project]\nname="other"\nversion="0.4.0"\n', + encoding="utf-8", + ) + + with self.assertRaises(LIFECYCLE.LifecycleFailure): + LIFECYCLE.project_identity(root, "scriptorium-suite") + + +class IsolationTests(unittest.TestCase): + def test_environment_drops_provider_and_index_credentials(self): + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + scripts = root / "clean-venv" / "Scripts" + with mock.patch.dict( + os.environ, + { + "PATH": "system-path", + "OPENAI_API_KEY": "private", + "PIP_INDEX_URL": "https://user:secret@example.invalid/simple", + }, + clear=True, + ): + env = LIFECYCLE.isolated_environment(root, scripts) + + self.assertNotIn("OPENAI_API_KEY", env) + self.assertNotIn("PIP_INDEX_URL", env) + self.assertEqual(env["PIP_NO_INDEX"], "1") + self.assertEqual(env["PIP_NO_INPUT"], "1") + self.assertTrue(env["PATH"].startswith(str(scripts))) + + def test_staging_excludes_unselected_private_directories(self): + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + source = root / "source" + package = source / "src" / "scriptorium" + package.mkdir(parents=True) + (source / "pyproject.toml").write_text( + '[project]\nname="scriptorium-suite"\nversion="0.4.0"\n', + encoding="utf-8", + ) + (source / "README.md").write_text("Public", encoding="utf-8") + (package / "__init__.py").write_text("__version__='0.4.0'\n", encoding="utf-8") + private = source / "private-memory" + private.mkdir() + (private / "session.jsonl").write_text("private", encoding="utf-8") + + staged = LIFECYCLE.stage_package( + source, + root / "staged", + "scriptorium-suite", + ) + + self.assertTrue((staged / "src" / "scriptorium" / "__init__.py").is_file()) + self.assertFalse((staged / "private-memory").exists()) + + +class TransitionTests(unittest.TestCase): + def test_only_older_release_pair_is_covered(self): + self.assertEqual( + LIFECYCLE.release_relation("0.3.0", "0.4.0"), + "previous-is-older", + ) + self.assertEqual(LIFECYCLE.release_relation("0.4.0", "0.4.0"), "same") + self.assertEqual( + LIFECYCLE.release_relation("0.5.0", "0.4.0"), + "previous-is-newer", + ) + self.assertEqual(LIFECYCLE.release_relation("local", "0.4.0"), "unsupported") + + +class StandaloneComponentTests(unittest.TestCase): + def test_provenance_is_read_only_and_steward_is_preview_first(self): + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + scripts = root / "scripts" + scripts.mkdir() + env = {"PATH": str(scripts)} + calls = [] + + def fake_run(command, *, stage, cwd, env, timeout=180): + arguments = [str(value) for value in command] + calls.append((stage, arguments, dict(env))) + if stage == "standalone-provenance": + self.assertEqual( + Path(env["PROVENANCE_HOME"]).name, + "standalone-provenance", + ) + if stage.startswith("standalone-steward") and "--run" in arguments: + vault = Path(arguments[arguments.index("--vault") + 1]) + dashboard = vault / "Projects" / "_总纲.md" + if not dashboard.exists(): + dashboard.write_text( + "<!-- steward:portfolio:begin -->\n" + "synthetic-system\n" + "<!-- steward:portfolio:end -->\n", + encoding="utf-8", + ) + return mock.Mock(returncode=0, stdout="", stderr="") + + with mock.patch.object(LIFECYCLE, "run", side_effect=fake_run): + LIFECYCLE.verify_standalone_components( + scripts, + root=root, + env=env, + ) + + stages = [stage for stage, _, _ in calls] + self.assertEqual( + stages, + [ + "standalone-provenance", + "standalone-steward-preview", + "standalone-steward-run", + "standalone-steward-idempotent", + ], + ) + + def test_rejects_a_provenance_status_write(self): + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + scripts = root / "scripts" + scripts.mkdir() + + def fake_run(command, *, stage, cwd, env, timeout=180): + if stage == "standalone-provenance": + (Path(env["PROVENANCE_HOME"]) / "unexpected.txt").write_text( + "unexpected", + encoding="utf-8", + ) + return mock.Mock(returncode=0, stdout="", stderr="") + + with mock.patch.object(LIFECYCLE, "run", side_effect=fake_run): + with self.assertRaises(LIFECYCLE.LifecycleFailure) as raised: + LIFECYCLE.verify_standalone_components( + scripts, + root=root, + env={"PATH": str(scripts)}, + ) + + self.assertEqual(raised.exception.stage, "standalone-provenance") + + +class ReportTests(unittest.TestCase): + def test_report_is_created_once_and_not_replaced(self): + with tempfile.TemporaryDirectory() as raw: + path = Path(raw) / "report.json" + payload = {"status": "passed"} + + LIFECYCLE.write_report(path, payload) + + self.assertEqual(json.loads(path.read_text(encoding="utf-8")), payload) + with self.assertRaises(LIFECYCLE.LifecycleFailure): + LIFECYCLE.write_report(path, {"status": "changed"}) + self.assertEqual(json.loads(path.read_text(encoding="utf-8")), payload) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_installer.py b/tests/test_installer.py new file mode 100644 index 0000000..1b84ac1 --- /dev/null +++ b/tests/test_installer.py @@ -0,0 +1,236 @@ +from __future__ import annotations + +import json +import hashlib +import tempfile +import unittest +import zipfile +from dataclasses import replace +from pathlib import Path +from unittest import mock + +from scriptorium.components import ComponentCatalog, load_component_catalog +from scriptorium.installer import ( + MAX_ASSET_BYTES, + InstallError, + execute_install, + plan_install, +) + + +class InstallerTests(unittest.TestCase): + def capture_catalog(self, asset: Path) -> ComponentCatalog: + catalog = load_component_catalog() + digest = hashlib.sha256(asset.read_bytes()).hexdigest() + components = dict(catalog.components) + components["capture"] = replace( + components["capture"], artifact_sha256=digest + ) + return replace(catalog, components=components) + + def test_preview_writes_nothing(self): + with tempfile.TemporaryDirectory() as temporary: + target = Path(temporary) / "suite components" + report = plan_install(profile="core", target=target) + self.assertEqual(report["status"], "planned") + self.assertEqual(report["writes"], "none") + self.assertEqual( + [action["component"] for action in report["actions"]], + ["scriptorium-spec", "provenance"], + ) + self.assertFalse(target.exists()) + + def test_capture_preview_is_honest_about_unpublished_asset(self): + with tempfile.TemporaryDirectory() as temporary: + target = Path(temporary) / "capture" + report = plan_install(profile="capture", target=target) + self.assertEqual(report["status"], "action-required") + self.assertEqual(report["summary"]["unavailable"], 1) + self.assertFalse(report["actions"][0]["available"]) + with self.assertRaisesRegex(InstallError, "unpublished"): + execute_install(profile="capture", target=target) + self.assertFalse(target.exists()) + + def test_downloaded_capture_asset_installs_without_cloning_provenance(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + asset = root / "scriptorium-local-conversation-export-1.0.0.zip" + with zipfile.ZipFile(asset, "w") as archive: + archive.writestr("manifest.json", '{"manifest_version": 3}') + archive.writestr("src/popup.html", "synthetic") + catalog = self.capture_catalog(asset) + target = root / "capture-only" + + preview = plan_install( + profile="capture", target=target, asset=asset, catalog=catalog + ) + self.assertEqual(preview["status"], "planned") + self.assertEqual(preview["network"], "not-required") + self.assertFalse(target.exists()) + + report = execute_install( + profile="capture", target=target, asset=asset, catalog=catalog + ) + self.assertEqual(report["status"], "installed") + self.assertTrue((target / "capture" / asset.name).is_file()) + self.assertEqual( + (target / "capture" / "unpacked" / "src" / "popup.html").read_text( + encoding="utf-8" + ), + "synthetic", + ) + self.assertFalse((target / "Provenance").exists()) + + def test_capture_asset_rejects_path_traversal(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + asset = root / "scriptorium-local-conversation-export-1.0.0.zip" + with zipfile.ZipFile(asset, "w") as archive: + archive.writestr("../outside.txt", "do not write") + catalog = self.capture_catalog(asset) + target = root / "capture-only" + with self.assertRaisesRegex(InstallError, "unsafe"): + execute_install( + profile="capture", target=target, asset=asset, catalog=catalog + ) + self.assertFalse((root / "outside.txt").exists()) + marker = json.loads( + (target / ".scriptorium-install.json").read_text(encoding="utf-8") + ) + self.assertEqual(marker["state"], "failed") + + def test_capture_asset_rejects_case_colliding_entries(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + asset = root / "scriptorium-local-conversation-export-1.0.0.zip" + with zipfile.ZipFile(asset, "w") as archive: + archive.writestr("src/popup.html", "first") + archive.writestr("SRC/popup.html", "second") + catalog = self.capture_catalog(asset) + with self.assertRaisesRegex(InstallError, "entry set"): + execute_install( + profile="capture", + target=root / "capture-only", + asset=asset, + catalog=catalog, + ) + + def test_capture_asset_is_rechecked_after_copy(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + asset = root / "scriptorium-local-conversation-export-1.0.0.zip" + with zipfile.ZipFile(asset, "w") as archive: + archive.writestr("manifest.json", '{}') + catalog = self.capture_catalog(asset) + + def corrupt_copy(_source, destination): + Path(destination).write_bytes(b"changed during copy") + + with mock.patch( + "scriptorium.installer.shutil.copyfile", side_effect=corrupt_copy + ): + with self.assertRaisesRegex(InstallError, "changed while copying"): + execute_install( + profile="capture", + target=root / "capture-only", + asset=asset, + catalog=catalog, + ) + + def test_capture_asset_rejects_oversized_archive_before_install(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + asset = root / "scriptorium-local-conversation-export-1.0.0.zip" + with asset.open("wb") as stream: + stream.seek(MAX_ASSET_BYTES) + stream.write(b"0") + target = root / "capture-only" + with self.assertRaisesRegex(InstallError, "too large"): + plan_install( + profile="capture", + target=target, + asset=asset, + catalog=load_component_catalog(), + ) + self.assertFalse(target.exists()) + + def test_nonempty_unowned_target_is_refused(self): + with tempfile.TemporaryDirectory() as temporary: + target = Path(temporary) / "existing" + target.mkdir() + sentinel = target / "user.txt" + sentinel.write_text("owned by user", encoding="utf-8") + with self.assertRaisesRegex(InstallError, "not owned"): + plan_install(profile="core", target=target) + self.assertEqual(sentinel.read_text(encoding="utf-8"), "owned by user") + + def test_malformed_ownership_marker_is_refused(self): + with tempfile.TemporaryDirectory() as temporary: + target = Path(temporary) / "existing" + target.mkdir() + (target / ".scriptorium-install.json").write_text( + '{"format_version": 1}\n', encoding="utf-8" + ) + with self.assertRaisesRegex(InstallError, "unsupported"): + plan_install(profile="core", target=target) + + def test_run_clones_exact_components_and_writes_marker_last(self): + catalog = load_component_catalog() + with tempfile.TemporaryDirectory() as temporary: + target = Path(temporary) / "managed" + + def fake_install(component, destination): + destination.mkdir(parents=True) + (destination / ".git").mkdir() + + with mock.patch( + "scriptorium.installer._install_source", side_effect=fake_install + ), mock.patch("scriptorium.installer._verify_checkout"): + report = execute_install( + profile="core", target=target, catalog=catalog + ) + + self.assertEqual(report["status"], "installed") + marker = json.loads( + (target / ".scriptorium-install.json").read_text(encoding="utf-8") + ) + self.assertEqual(marker["profile"], "core") + self.assertEqual( + set(marker["components"]), {"scriptorium-spec", "provenance"} + ) + powershell = (target / "scriptorium-env.ps1").read_text(encoding="utf-8") + bash = (target / "scriptorium-env.sh").read_text(encoding="utf-8") + self.assertIn("SCRIPTORIUM_SPEC_ROOT", powershell) + self.assertIn("SCRIPTORIUM_PROVENANCE_ROOT", powershell) + self.assertNotIn(str(target), powershell) + self.assertIn("SCRIPTORIUM_COMPONENTS_ROOT", bash) + + def test_failed_run_keeps_a_failed_ownership_marker_without_deleting(self): + with tempfile.TemporaryDirectory() as temporary: + target = Path(temporary) / "managed" + target.mkdir() + + def fail_after_create(component, destination): + destination.mkdir(parents=True) + raise InstallError("synthetic failure", code="command_failed") + + with mock.patch( + "scriptorium.installer._install_source", side_effect=fail_after_create + ): + with self.assertRaisesRegex(InstallError, "synthetic"): + execute_install(profile="core", target=target) + + self.assertTrue(target.is_dir()) + self.assertTrue((target / "scriptorium-spec").is_dir()) + marker = json.loads( + (target / ".scriptorium-install.json").read_text(encoding="utf-8") + ) + self.assertEqual(marker["state"], "failed") + + def test_user_home_is_too_broad_for_component_install(self): + with self.assertRaisesRegex(InstallError, "user home"): + plan_install(profile="core", target=Path.home()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_migration.py b/tests/test_migration.py new file mode 100644 index 0000000..3062e03 --- /dev/null +++ b/tests/test_migration.py @@ -0,0 +1,945 @@ +import copy +import errno +import json +import os +import stat +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +from scriptorium import migration as migration_module +from scriptorium.migration import ( + MANIFEST_PRIVACY, + MANIFEST_VERSION, + MigrationError, + apply_migration, + plan_migration, + rollback_migration, +) + + +def snapshot(root: Path) -> dict[str, bytes | None]: + result: dict[str, bytes | None] = {} + for path in sorted(root.rglob("*")): + relative = path.relative_to(root).as_posix() + result[relative] = path.read_bytes() if path.is_file() else None + return result + + +class MigrationTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.base = Path(self.temporary.name).resolve() + self.workspace = self.base / "research workspace" + self.sources = self.base / "selected sources" + self.platform_state = self.base / "platform state" + self.environment = mock.patch.dict( + os.environ, + { + "LOCALAPPDATA": str(self.platform_state), + "XDG_STATE_HOME": str(self.platform_state), + }, + clear=False, + ) + self.environment.start() + self.state_root = ( + self.platform_state / "Scriptorium" / "state" + if os.name == "nt" + else self.platform_state / "scriptorium" + ) + self.workspace.mkdir() + self.sources.mkdir() + + def tearDown(self): + self.environment.stop() + self.temporary.cleanup() + + def write_sources(self): + (self.sources / "notes").mkdir() + markdown = self.sources / "notes" / "idea.md" + pdf = self.sources / "paper.pdf" + ignored = self.sources / "raw.txt" + markdown.write_text("# Synthetic idea\n", encoding="utf-8") + pdf.write_bytes(b"%PDF-1.4\nsynthetic\n") + ignored.write_text("not selected by suffix", encoding="utf-8") + return markdown, pdf, ignored + + def plan(self, sources=None, *, batch_id="batch-001", state_root=None): + return plan_migration( + sources if sources is not None else [self.sources], + workspace=self.workspace, + batch_id=batch_id, + state_root=self.state_root if state_root is None else state_root, + ) + + def target(self, plan, index=0) -> Path: + return Path(plan.manifest["entries"][index]["target"]) + + def manifest_path(self, plan) -> Path: + return migration_module._manifest_path(plan.manifest) + + def test_plan_is_read_only_private_and_repr_is_path_free(self): + self.write_sources() + workspace_before = snapshot(self.workspace) + sources_before = snapshot(self.sources) + + planned = self.plan() + + self.assertEqual(snapshot(self.workspace), workspace_before) + self.assertEqual(snapshot(self.sources), sources_before) + self.assertFalse(self.state_root.exists()) + self.assertEqual(planned.report["status"], "planned") + self.assertEqual(planned.report["summary"]["files"], 2) + self.assertEqual(planned.report["summary"]["markdown"], 1) + self.assertEqual(planned.report["summary"]["pdf"], 1) + self.assertNotIn(str(self.base), json.dumps(planned.report)) + self.assertNotIn(str(self.base), repr(planned)) + self.assertEqual(planned.manifest["schema_version"], MANIFEST_VERSION) + self.assertEqual(planned.manifest["privacy"], MANIFEST_PRIVACY) + self.assertEqual(Path(planned.manifest["state_root"]), self.state_root) + + def test_default_state_root_uses_platform_local_state_outside_workspace(self): + source = self.sources / "idea.md" + source.write_text("synthetic", encoding="utf-8") + local_state = self.base / "platform state" + environment = { + "LOCALAPPDATA": str(local_state), + "XDG_STATE_HOME": str(local_state), + } + with mock.patch.dict(os.environ, environment, clear=False): + planned = plan_migration( + [source], workspace=self.workspace, batch_id="default-state" + ) + expected = ( + local_state / "Scriptorium" / "state" + if os.name == "nt" + else local_state / "scriptorium" + ) + self.assertEqual(Path(planned.manifest["state_root"]), expected) + self.assertFalse(expected.exists()) + + def test_noncanonical_explicit_state_root_is_rejected_without_path(self): + source = self.sources / "idea.md" + source.write_text("synthetic", encoding="utf-8") + with self.assertRaises(MigrationError) as raised: + self.plan([source], state_root=self.workspace / ".private") + self.assertEqual(raised.exception.code, "noncanonical_state_root") + self.assertNotIn(str(self.base), str(raised.exception)) + + def test_canonical_state_root_inside_workspace_is_rejected(self): + source = self.sources / "idea.md" + source.write_text("synthetic", encoding="utf-8") + local_state = self.workspace / "platform state" + with mock.patch.dict( + os.environ, + { + "LOCALAPPDATA": str(local_state), + "XDG_STATE_HOME": str(local_state), + }, + clear=False, + ): + with self.assertRaises(MigrationError) as raised: + plan_migration( + [source], workspace=self.workspace, batch_id="overlap" + ) + self.assertEqual(raised.exception.code, "state_workspace_overlap") + + def test_canonical_state_root_inside_source_root_is_rejected(self): + source = self.sources / "idea.md" + source.write_text("synthetic", encoding="utf-8") + with mock.patch.dict( + os.environ, + { + "LOCALAPPDATA": str(self.sources), + "XDG_STATE_HOME": str(self.sources), + }, + clear=False, + ): + with self.assertRaises(MigrationError) as raised: + plan_migration( + [self.sources], + workspace=self.workspace, + batch_id="source-overlap", + ) + self.assertEqual(raised.exception.code, "state_root_in_source") + + def test_source_directory_inside_canonical_state_root_is_rejected(self): + source = self.state_root / "candidate inputs" + source.mkdir(parents=True) + (source / "idea.md").write_text("synthetic", encoding="utf-8") + with self.assertRaises(MigrationError) as raised: + self.plan([source]) + self.assertEqual(raised.exception.code, "source_state_overlap") + self.assertFalse((self.workspace / "Sources").exists()) + + def test_source_file_inside_canonical_state_root_is_rejected(self): + source = self.state_root / "candidate.md" + source.parent.mkdir(parents=True) + source.write_text("synthetic", encoding="utf-8") + with self.assertRaises(MigrationError) as raised: + self.plan([source]) + self.assertEqual(raised.exception.code, "source_state_overlap") + self.assertFalse((self.workspace / "Sources").exists()) + + def test_apply_copies_supported_files_and_keeps_manifest_outside_workspace(self): + self.write_sources() + sources_before = snapshot(self.sources) + applied = apply_migration(self.plan()) + + self.assertEqual(applied.report["status"], "applied") + self.assertEqual(applied.report["summary"]["changed"], 2) + self.assertEqual(snapshot(self.sources), sources_before) + for entry in applied.manifest["entries"]: + self.assertEqual( + Path(entry["target"]).read_bytes(), Path(entry["source"]).read_bytes() + ) + self.assertEqual(entry["state"], "created") + self.assertEqual( + entry["file_identity"], + migration_module._file_identity(Path(entry["target"])), + ) + self.assertTrue(self.manifest_path(applied).is_file()) + self.assertFalse((self.workspace / ".scriptorium").exists()) + self.assertNotIn(str(self.base), json.dumps(applied.report)) + self.assertNotIn(str(self.base), repr(applied)) + + def test_publication_uses_same_directory_hard_link_create(self): + source = self.sources / "idea.md" + source.write_text("synthetic", encoding="utf-8") + planned = self.plan([source]) + target = self.target(planned) + + with mock.patch( + "scriptorium.migration.os.link", wraps=os.link + ) as link: + apply_migration(planned) + + self.assertEqual(link.call_count, 1) + stage, published = map(Path, link.call_args.args) + self.assertEqual(published, target) + self.assertEqual(stage.parent, target.parent) + self.assertTrue(stage.is_file()) + self.assertTrue(os.path.samefile(stage, target)) + + def test_existing_target_is_never_adopted_or_overwritten(self): + source = self.sources / "idea.md" + source.write_text("synthetic", encoding="utf-8") + planned = self.plan([source]) + destination = self.target(planned) + destination.parent.mkdir(parents=True) + destination.write_text("user owned", encoding="utf-8") + + with self.assertRaises(MigrationError) as raised: + apply_migration(planned) + + self.assertEqual(raised.exception.code, "target_exists") + self.assertEqual(destination.read_text(encoding="utf-8"), "user owned") + self.assertFalse(self.manifest_path(planned).exists()) + + def test_hard_link_failure_is_safe_and_retryable(self): + source = self.sources / "idea.md" + source.write_text("synthetic", encoding="utf-8") + planned = self.plan([source]) + target = self.target(planned) + + with mock.patch( + "scriptorium.migration.os.link", + side_effect=OSError(errno.EPERM, "synthetic"), + ): + with self.assertRaises(MigrationError) as raised: + apply_migration(planned) + + self.assertEqual(raised.exception.code, "atomic_publish_unavailable") + self.assertFalse(target.exists()) + recovered = apply_migration(planned) + self.assertEqual(recovered.report["status"], "applied") + self.assertEqual(target.read_text(encoding="utf-8"), "synthetic") + + def test_cross_volume_publish_error_has_no_overwrite_fallback(self): + source = self.sources / "idea.md" + source.write_text("synthetic", encoding="utf-8") + planned = self.plan([source]) + + with mock.patch( + "scriptorium.migration.os.link", + side_effect=OSError(errno.EXDEV, "synthetic"), + ): + with self.assertRaises(MigrationError) as raised: + apply_migration(planned) + + self.assertEqual( + raised.exception.code, "cross_volume_publish_unsupported" + ) + self.assertFalse(self.target(planned).exists()) + + def test_source_change_is_rejected_before_target_write(self): + markdown, _, _ = self.write_sources() + planned = self.plan() + markdown.write_text("changed after preview\n", encoding="utf-8") + + with self.assertRaises(MigrationError) as raised: + apply_migration(planned) + + self.assertEqual(raised.exception.code, "source_hash_changed") + self.assertFalse((self.workspace / "Sources").exists()) + self.assertFalse(self.manifest_path(planned).exists()) + + def test_repeat_apply_stays_idempotent_when_sources_are_gone(self): + self.write_sources() + planned = self.plan() + first = apply_migration(planned) + target_mtimes = { + Path(entry["target"]): Path(entry["target"]).stat().st_mtime_ns + for entry in first.manifest["entries"] + } + manifest_path = self.manifest_path(first) + manifest_mtime = manifest_path.stat().st_mtime_ns + for path in sorted(self.sources.rglob("*"), reverse=True): + if path.is_file(): + path.unlink() + else: + path.rmdir() + + second = apply_migration(planned) + + self.assertEqual(second.report["status"], "unchanged") + self.assertEqual(second.report["summary"]["changed"], 0) + self.assertEqual(second.report["summary"]["unchanged"], 2) + self.assertEqual(manifest_path.stat().st_mtime_ns, manifest_mtime) + self.assertEqual( + {path: path.stat().st_mtime_ns for path in target_mtimes}, target_mtimes + ) + + def test_new_process_can_load_verify_reapply_and_rollback_without_sources(self): + self.write_sources() + planned = self.plan(batch_id="process-recovery") + applied = apply_migration(planned) + targets = [Path(entry["target"]) for entry in applied.manifest["entries"]] + for path in sorted(self.sources.rglob("*"), reverse=True): + if path.is_file(): + path.unlink() + else: + path.rmdir() + + source_root = Path(__file__).resolve().parents[1] / "src" + code = """ +import json +import sys +from scriptorium.migration import ( + load_migration, + reapply_migration, + rollback_migration, + verify_migration, +) +workspace, batch_id = sys.argv[1:3] +loaded = load_migration(workspace=workspace, batch_id=batch_id) +verified = verify_migration(workspace=workspace, batch_id=batch_id) +reapplied = reapply_migration(workspace=workspace, batch_id=batch_id) +rolled_back = rollback_migration( + load_migration(workspace=workspace, batch_id=batch_id) +) +print(json.dumps({ + "load": loaded.report["status"], + "verify": verified.report["status"], + "reapply": reapplied.report["status"], + "rollback": rolled_back.report["status"], +})) +""" + environment = dict(os.environ) + environment["PYTHONPATH"] = os.pathsep.join( + [str(source_root), environment.get("PYTHONPATH", "")] + ) + completed = subprocess.run( + [ + sys.executable, + "-c", + code, + str(self.workspace), + "process-recovery", + ], + env=environment, + capture_output=True, + text=True, + timeout=15, + check=False, + ) + + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual( + json.loads(completed.stdout), + { + "load": "applied", + "verify": "applied", + "reapply": "unchanged", + "rollback": "rolled-back", + }, + ) + self.assertNotIn(str(self.base), completed.stdout) + self.assertNotIn(str(self.base), completed.stderr) + self.assertTrue(all(not target.exists() for target in targets)) + + def test_runtime_state_tampering_is_covered_by_integrity(self): + source = self.sources / "idea.md" + source.write_text("synthetic", encoding="utf-8") + manifest = copy.deepcopy(self.plan([source]).manifest) + manifest["run_state"] = "applied" + + with self.assertRaises(MigrationError) as raised: + apply_migration(manifest) + + self.assertEqual(raised.exception.code, "manifest_integrity_failed") + + def test_invalid_but_resealed_state_combination_is_rejected(self): + source = self.sources / "idea.md" + source.write_text("synthetic", encoding="utf-8") + manifest = copy.deepcopy(self.plan([source]).manifest) + manifest["run_state"] = "applied" + migration_module._seal_manifest(manifest) + + with self.assertRaises(MigrationError) as raised: + apply_migration(manifest) + + self.assertEqual(raised.exception.code, "invalid_state_transition") + + def test_tampered_escape_is_rejected_even_when_digests_are_recomputed(self): + source = self.sources / "idea.md" + source.write_text("synthetic", encoding="utf-8") + manifest = copy.deepcopy(self.plan([source]).manifest) + manifest["entries"][0]["relative_target"] = "../outside.md" + manifest["entries"][0]["target"] = str(self.base / "outside.md") + migration_module._seal_manifest(manifest, new_plan=True) + + with self.assertRaises(MigrationError) as raised: + apply_migration(manifest) + + self.assertEqual(raised.exception.code, "target_escape") + self.assertFalse((self.base / "outside.md").exists()) + + def test_rollback_removes_owned_files_only_and_is_idempotent(self): + self.write_sources() + planned = self.plan() + applied = apply_migration(planned) + sibling = self.workspace / "Sources" / "Imported" / "user-owned.md" + sibling.write_text("keep", encoding="utf-8") + anchors = [ + migration_module._stage(applied.manifest, entry) + for entry in applied.manifest["entries"] + ] + self.assertTrue(all(anchor.is_file() for anchor in anchors)) + + rolled_back = rollback_migration(planned) + + self.assertEqual(rolled_back.report["status"], "rolled-back") + self.assertEqual(rolled_back.report["summary"]["changed"], 2) + self.assertTrue(sibling.is_file()) + self.assertTrue( + all(not Path(entry["target"]).exists() for entry in applied.manifest["entries"]) + ) + self.assertTrue(all(not anchor.exists() for anchor in anchors)) + repeated = rollback_migration(planned) + self.assertEqual(repeated.report["status"], "unchanged") + self.assertTrue(sibling.is_file()) + self.assertNotIn(str(self.base), json.dumps(rolled_back.report)) + + def test_rollback_persists_delete_intent_and_recovers_after_state_write_failure(self): + self.write_sources() + planned = self.plan() + applied = apply_migration(planned) + targets = [Path(entry["target"]) for entry in applied.manifest["entries"]] + real_persist = migration_module._persist_manifest + failed = False + + def fail_after_first_delete(manifest): + nonlocal failed + if ( + not failed + and manifest["run_state"] == "rolling-back" + and manifest["entries"][0]["state"] == "deleted" + ): + failed = True + raise MigrationError("state_write_failed") + return real_persist(manifest) + + with mock.patch( + "scriptorium.migration._persist_manifest", + side_effect=fail_after_first_delete, + ): + with self.assertRaises(MigrationError) as raised: + rollback_migration(applied) + + self.assertEqual(raised.exception.code, "state_write_failed") + self.assertFalse(targets[0].exists()) + stored = json.loads(self.manifest_path(planned).read_text(encoding="utf-8")) + self.assertEqual(stored["entries"][0]["state"], "delete-pending") + + recovered = rollback_migration(planned) + self.assertEqual(recovered.report["status"], "rolled-back") + self.assertTrue(all(not target.exists() for target in targets)) + + def test_modified_target_fails_closed_before_any_rollback_deletion(self): + self.write_sources() + applied = apply_migration(self.plan()) + targets = [Path(entry["target"]) for entry in applied.manifest["entries"]] + targets[-1].write_bytes(b"user modification") + + with self.assertRaises(MigrationError) as raised: + rollback_migration(applied) + + self.assertEqual(raised.exception.code, "owned_target_changed") + self.assertTrue(all(path.exists() for path in targets)) + + def test_same_byte_replacement_is_not_deleted_by_rollback(self): + source = self.sources / "idea.md" + source.write_bytes(b"synthetic replacement sentinel") + applied = apply_migration(self.plan([source])) + entry = applied.manifest["entries"][0] + target = Path(entry["target"]) + anchor = migration_module._stage(applied.manifest, entry) + self.assertTrue(os.path.samefile(anchor, target)) + + target.unlink() + target.write_bytes(source.read_bytes()) + self.assertFalse(os.path.samefile(anchor, target)) + + with self.assertRaises(MigrationError) as raised: + rollback_migration(applied) + + self.assertEqual(raised.exception.code, "owned_target_changed") + self.assertEqual(target.read_bytes(), source.read_bytes()) + self.assertTrue(anchor.is_file()) + + def test_target_replaced_after_preflight_is_not_deleted(self): + source = self.sources / "idea.md" + source.write_bytes(b"synthetic post-preflight sentinel") + applied = apply_migration(self.plan([source])) + entry = applied.manifest["entries"][0] + target = Path(entry["target"]) + anchor = migration_module._stage(applied.manifest, entry) + real_preflight = migration_module._preflight_rollback + replaced = False + + def replace_after_preflight(manifest, **kwargs): + nonlocal replaced + real_preflight(manifest, **kwargs) + if not replaced: + replaced = True + target.unlink() + target.write_bytes(source.read_bytes()) + + with mock.patch( + "scriptorium.migration._preflight_rollback", + side_effect=replace_after_preflight, + ): + with self.assertRaises(MigrationError) as raised: + rollback_migration(applied) + + self.assertEqual(raised.exception.code, "owned_target_changed") + self.assertEqual(target.read_bytes(), source.read_bytes()) + self.assertTrue(anchor.is_file()) + self.assertFalse(os.path.samefile(anchor, target)) + self.assertFalse( + any(target.parent.glob(".scriptorium-*.rollback")) + ) + + def test_preoccupied_quarantine_never_overwrites_or_deletes_either_file(self): + source = self.sources / "idea.md" + source.write_bytes(b"synthetic quarantine source") + applied = apply_migration(self.plan([source])) + entry = applied.manifest["entries"][0] + target = Path(entry["target"]) + anchor = migration_module._stage(applied.manifest, entry) + token = "2" * 32 + quarantine = target.parent / ( + f".scriptorium-{applied.manifest['batch_id']}-{entry['id']}-" + f"{token}.rollback" + ) + quarantine.write_bytes(b"user quarantine occupant") + + with mock.patch( + "scriptorium.migration.secrets.token_hex", return_value=token + ): + with self.assertRaises(MigrationError) as raised: + rollback_migration(applied) + + self.assertEqual(raised.exception.code, "quarantine_conflict") + self.assertEqual(target.read_bytes(), source.read_bytes()) + self.assertEqual(quarantine.read_bytes(), b"user quarantine occupant") + self.assertTrue(anchor.is_file()) + + def test_restore_blocked_preserves_moved_replacement_and_new_target(self): + source = self.sources / "idea.md" + source.write_bytes(b"synthetic restore source") + applied = apply_migration(self.plan([source])) + entry = applied.manifest["entries"][0] + target = Path(entry["target"]) + anchor = migration_module._stage(applied.manifest, entry) + real_preflight = migration_module._preflight_rollback + real_rename = migration_module._rename_noreplace + replaced = False + reoccupied = False + + def replace_after_preflight(manifest, **kwargs): + nonlocal replaced + real_preflight(manifest, **kwargs) + if not replaced: + replaced = True + target.unlink() + target.write_bytes(source.read_bytes()) + + def reoccupy_after_move(old, new): + nonlocal reoccupied + real_rename(old, new) + if old == target and not reoccupied: + reoccupied = True + target.write_bytes(b"new target occupant") + + with mock.patch( + "scriptorium.migration._preflight_rollback", + side_effect=replace_after_preflight, + ), mock.patch( + "scriptorium.migration._rename_noreplace", + side_effect=reoccupy_after_move, + ): + with self.assertRaises(MigrationError) as raised: + rollback_migration(applied) + + self.assertEqual(raised.exception.code, "rollback_restore_blocked") + self.assertEqual(target.read_bytes(), b"new target occupant") + stored = migration_module.load_migration( + workspace=self.workspace, batch_id="batch-001" + ).manifest + quarantine = migration_module._quarantine( + stored, stored["entries"][0] + ) + self.assertEqual(quarantine.read_bytes(), source.read_bytes()) + self.assertTrue(anchor.is_file()) + + def test_random_stage_collision_preserves_the_competing_file(self): + source = self.sources / "idea.md" + source.write_bytes(b"synthetic collision source") + planned = self.plan([source]) + entry = planned.manifest["entries"][0] + target = Path(entry["target"]) + target.parent.mkdir(parents=True) + collision_name = ( + f".scriptorium-{planned.manifest['batch_id']}-{entry['id']}-" + f"{'0' * 32}.stage" + ) + collision = target.parent / collision_name + collision.write_bytes(b"user replacement") + + with mock.patch( + "scriptorium.migration.secrets.token_hex", + side_effect=["0" * 32, "1" * 32], + ): + applied = apply_migration(planned) + + self.assertEqual(collision.read_bytes(), b"user replacement") + anchor = migration_module._stage( + applied.manifest, applied.manifest["entries"][0] + ) + self.assertNotEqual(anchor, collision) + self.assertEqual(anchor.read_bytes(), source.read_bytes()) + + def test_complete_unclaimed_stage_after_state_crash_does_not_block_retry(self): + source = self.sources / "idea.md" + source.write_bytes(b"synthetic unclaimed stage") + planned = self.plan([source]) + real_persist = migration_module._persist_manifest + failed = False + + def fail_before_stage_claim(manifest): + nonlocal failed + entry = manifest["entries"][0] + if ( + not failed + and entry["state"] == "creating" + and entry["stage_name"] is not None + ): + failed = True + raise MigrationError("state_write_failed") + return real_persist(manifest) + + with mock.patch( + "scriptorium.migration._persist_manifest", + side_effect=fail_before_stage_claim, + ): + with self.assertRaises(MigrationError) as raised: + apply_migration(planned) + + self.assertEqual(raised.exception.code, "state_write_failed") + target_parent = self.target(planned).parent + orphaned = list(target_parent.glob(".scriptorium-*.stage")) + self.assertEqual(len(orphaned), 1) + + recovered = apply_migration(planned) + + anchor = migration_module._stage( + recovered.manifest, recovered.manifest["entries"][0] + ) + self.assertNotEqual(anchor, orphaned[0]) + self.assertEqual(orphaned[0].read_bytes(), source.read_bytes()) + self.assertEqual(anchor.read_bytes(), source.read_bytes()) + + def test_stage_replaced_after_identity_persist_is_rejected_and_preserved(self): + source = self.sources / "idea.md" + source.write_bytes(b"synthetic claimed stage") + planned = self.plan([source]) + real_persist = migration_module._persist_manifest + replaced = False + replacement = None + + def replace_after_stage_claim(manifest): + nonlocal replaced, replacement + real_persist(manifest) + entry = manifest["entries"][0] + if ( + not replaced + and entry["state"] == "creating" + and entry["stage_name"] is not None + ): + replaced = True + replacement = migration_module._stage(manifest, entry) + foreign = replacement.with_name(f"{replacement.name}.foreign") + foreign.write_bytes(source.read_bytes()) + self.assertNotEqual( + entry["file_identity"], migration_module._file_identity(foreign) + ) + os.replace(foreign, replacement) + + with mock.patch( + "scriptorium.migration._persist_manifest", + side_effect=replace_after_stage_claim, + ): + with self.assertRaises(MigrationError) as raised: + apply_migration(planned) + + self.assertEqual(raised.exception.code, "owned_target_changed") + self.assertIsNotNone(replacement) + self.assertEqual(replacement.read_bytes(), source.read_bytes()) + self.assertFalse(self.target(planned).exists()) + + def test_hash_mismatch_preserves_unclaimed_stage_for_manual_review(self): + source = self.sources / "idea.md" + source.write_bytes(b"synthetic original") + planned = self.plan([source]) + real_verify = migration_module._verify_source + calls = 0 + + def change_after_second_verification(entry): + nonlocal calls + real_verify(entry) + calls += 1 + if calls == 2: + source.write_bytes(b"synthetic changed") + + with mock.patch( + "scriptorium.migration._verify_source", + side_effect=change_after_second_verification, + ): + with self.assertRaises(MigrationError) as raised: + apply_migration(planned) + + self.assertEqual(raised.exception.code, "source_hash_changed") + target = self.target(planned) + self.assertFalse(target.exists()) + orphaned = list(target.parent.glob(".scriptorium-*.stage")) + self.assertEqual(len(orphaned), 1) + self.assertEqual(orphaned[0].read_bytes(), b"synthetic changed") + + def test_changed_stage_is_not_deleted_during_apply_recovery(self): + source = self.sources / "idea.md" + source.write_bytes(b"synthetic stage sentinel") + planned = self.plan([source]) + with mock.patch( + "scriptorium.migration.os.link", + side_effect=OSError(errno.EPERM, "synthetic"), + ): + with self.assertRaises(MigrationError): + apply_migration(planned) + + stored = json.loads(self.manifest_path(planned).read_text(encoding="utf-8")) + entry = stored["entries"][0] + anchor = migration_module._stage(stored, entry) + anchor.unlink() + anchor.write_bytes(b"user replacement") + + with self.assertRaises(MigrationError) as raised: + apply_migration(planned) + + self.assertEqual(raised.exception.code, "staging_conflict") + self.assertEqual(anchor.read_bytes(), b"user replacement") + + def test_delete_pending_same_byte_anchor_replacement_is_not_deleted(self): + source = self.sources / "idea.md" + source.write_bytes(b"synthetic anchor sentinel") + applied = apply_migration(self.plan([source])) + current = copy.deepcopy(applied.manifest) + current["run_state"] = "rolling-back" + pending = current["entries"][0] + pending["state"] = "delete-pending" + pending["rollback_phase"] = "anchor" + pending["quarantine_name"] = migration_module._new_internal_name( + current, pending, "rollback" + ) + migration_module._persist_manifest(current) + + entry = current["entries"][0] + target = Path(entry["target"]) + anchor = migration_module._stage(current, entry) + foreign = anchor.with_name(f"{anchor.name}.foreign") + foreign.write_bytes(source.read_bytes()) + target.unlink() + os.replace(foreign, anchor) + self.assertNotEqual( + entry["file_identity"], migration_module._file_identity(anchor) + ) + + with self.assertRaises(MigrationError) as raised: + rollback_migration(current) + + self.assertEqual(raised.exception.code, "owned_target_changed") + self.assertEqual(anchor.read_bytes(), source.read_bytes()) + + def test_delete_pending_crash_recovers_with_persisted_anchor_identity(self): + source = self.sources / "idea.md" + source.write_bytes(b"synthetic crash sentinel") + applied = apply_migration(self.plan([source])) + current = copy.deepcopy(applied.manifest) + current["run_state"] = "rolling-back" + pending = current["entries"][0] + pending["state"] = "delete-pending" + pending["rollback_phase"] = "anchor" + pending["quarantine_name"] = migration_module._new_internal_name( + current, pending, "rollback" + ) + migration_module._persist_manifest(current) + + entry = current["entries"][0] + target = Path(entry["target"]) + anchor = migration_module._stage(current, entry) + target.unlink() + self.assertTrue(anchor.is_file()) + self.assertEqual( + entry["file_identity"], migration_module._file_identity(anchor) + ) + + recovered = rollback_migration(current) + + self.assertEqual(recovered.report["status"], "rolled-back") + self.assertFalse(target.exists()) + self.assertFalse(anchor.exists()) + + def test_each_persisted_quarantine_phase_recovers_after_crash(self): + phases = ( + "target", + "target-quarantined", + "anchor", + "anchor-quarantined", + ) + for index, phase in enumerate(phases, start=1): + with self.subTest(phase=phase): + source = self.sources / f"phase-{index}.md" + source.write_bytes(f"synthetic {phase}".encode()) + applied = apply_migration( + self.plan([source], batch_id=f"phase-{index}") + ) + current = copy.deepcopy(applied.manifest) + current["run_state"] = "rolling-back" + entry = current["entries"][0] + entry["state"] = "delete-pending" + entry["rollback_phase"] = phase + entry["quarantine_name"] = migration_module._new_internal_name( + current, entry, "rollback" + ) + target = Path(entry["target"]) + anchor = migration_module._stage(current, entry) + quarantine = migration_module._quarantine(current, entry) + migration_module._persist_manifest(current) + + if phase.startswith("anchor"): + target.unlink() + migration_module._rename_noreplace(anchor, quarantine) + else: + migration_module._rename_noreplace(target, quarantine) + + recovered = rollback_migration(current) + + self.assertEqual(recovered.report["status"], "rolled-back") + self.assertFalse(target.exists()) + self.assertFalse(anchor.exists()) + self.assertFalse(quarantine.exists()) + + def test_directory_without_supported_files_is_an_error(self): + (self.sources / "data.txt").write_text("synthetic", encoding="utf-8") + with self.assertRaises(MigrationError) as raised: + self.plan() + self.assertEqual(raised.exception.code, "no_supported_files") + self.assertEqual(snapshot(self.workspace), {}) + + def test_unsupported_explicit_file_is_rejected(self): + unsupported = self.sources / "data.csv" + unsupported.write_text("a,b\n1,2\n", encoding="utf-8") + with self.assertRaises(MigrationError) as raised: + self.plan([unsupported]) + self.assertEqual(raised.exception.code, "unsupported_source_type") + + def test_file_and_directory_selection_deduplicates_same_source(self): + markdown, _, _ = self.write_sources() + planned = self.plan([self.sources, markdown]) + sources = [entry["source"] for entry in planned.manifest["entries"]] + self.assertEqual(len(sources), len(set(sources))) + self.assertEqual(len(sources), 2) + + def test_reparse_attribute_is_rejected(self): + metadata = SimpleNamespace( + st_mode=stat.S_IFDIR, st_file_attributes=0x400 + ) + self.assertTrue(migration_module._linklike(metadata)) + + def test_nested_directory_symlink_is_rejected_when_supported(self): + outside = self.base / "outside" + outside.mkdir() + (outside / "private.md").write_text("private", encoding="utf-8") + linked = self.sources / "linked" + try: + linked.symlink_to(outside, target_is_directory=True) + except OSError as exc: + self.skipTest(f"directory symlinks are unavailable: {exc}") + with self.assertRaises(MigrationError) as raised: + self.plan() + self.assertEqual(raised.exception.code, "link_or_reparse_rejected") + + def test_kernel_lock_is_released_when_process_exits_without_cleanup(self): + lock = self.state_root / "migrations" / ("a" * 20) / "exit.lock" + source_root = Path(__file__).resolve().parents[1] / "src" + code = ( + "import os,sys;" + "from pathlib import Path;" + "from scriptorium.migration import _kernel_lock;" + "lock=Path(sys.argv[1]);" + "ctx=_kernel_lock(lock);" + "ctx.__enter__();" + "os._exit(0)" + ) + environment = dict(os.environ) + environment["PYTHONPATH"] = os.pathsep.join( + [str(source_root), environment.get("PYTHONPATH", "")] + ) + completed = subprocess.run( + [sys.executable, "-c", code, str(lock)], + env=environment, + timeout=10, + check=False, + ) + self.assertEqual(completed.returncode, 0) + with migration_module._kernel_lock(lock): + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_pull.py b/tests/test_pull.py index 9b69fe0..1e7fc64 100644 --- a/tests/test_pull.py +++ b/tests/test_pull.py @@ -29,7 +29,7 @@ def component_report( summary["codex_found"] = 1 return { "format_version": 1, - "generated_by": {"name": "provenance", "version": "0.17.0"}, + "generated_by": {"name": "provenance", "version": "0.18.0"}, "operation": "pull", "mode": mode, "status": status, @@ -338,7 +338,7 @@ def test_invalid_provenance_environment_root_does_not_fall_back_to_path(self): with self.assertRaisesRegex( PullError, "configured Provenance root is unavailable" ): - _resolve_provenance(None, expected_version="0.17.0") + _resolve_provenance(None, expected_version="0.18.0") installed.assert_not_called() def test_environment_is_narrow_and_raw_stderr_is_suppressed(self): @@ -444,14 +444,14 @@ def test_explicit_source_version_is_fixed_by_compatibility_manifest(self): encoding="utf-8", ) with self.assertRaisesRegex(PullError, "incompatible Provenance"): - _resolve_provenance(root, expected_version="0.17.0") + _resolve_provenance(root, expected_version="0.18.0") def test_non_json_extra_stdout_and_timeout_fail_closed(self): with self.assertRaisesRegex(PullError, "non-JSON or extra stdout"): _parse_component_report( json.dumps(component_report()) + "\nlog line", expected_mode="preview", - expected_version="0.17.0", + expected_version="0.18.0", process_exit_code=0, ) @@ -499,18 +499,18 @@ def test_report_exit_one_is_trusted_when_process_and_status_match(self): report, producer = _parse_component_report( json.dumps(component_report(status="blocked", exit_code=1)), expected_mode="preview", - expected_version="0.17.0", + expected_version="0.18.0", process_exit_code=1, ) self.assertEqual(report["exit_code"], 1) - self.assertEqual(producer["version"], "0.17.0") + self.assertEqual(producer["version"], "0.18.0") def test_report_mismatch_or_wrong_producer_is_rejected(self): with self.assertRaisesRegex(PullError, "exit code disagrees"): _parse_component_report( json.dumps(component_report()), expected_mode="preview", - expected_version="0.17.0", + expected_version="0.18.0", process_exit_code=1, ) wrong = component_report() @@ -519,7 +519,7 @@ def test_report_mismatch_or_wrong_producer_is_rejected(self): _parse_component_report( json.dumps(wrong), expected_mode="preview", - expected_version="0.17.0", + expected_version="0.18.0", process_exit_code=0, ) @@ -530,7 +530,7 @@ def test_report_types_are_strict_and_unknown_fields_are_dropped(self): _parse_component_report( json.dumps(boolean_version), expected_mode="preview", - expected_version="0.17.0", + expected_version="0.18.0", process_exit_code=0, ) @@ -540,7 +540,7 @@ def test_report_types_are_strict_and_unknown_fields_are_dropped(self): _parse_component_report( json.dumps(non_string_status), expected_mode="preview", - expected_version="0.17.0", + expected_version="0.18.0", process_exit_code=0, ) @@ -550,7 +550,7 @@ def test_report_types_are_strict_and_unknown_fields_are_dropped(self): _parse_component_report( json.dumps(non_integer_exit), expected_mode="preview", - expected_version="0.17.0", + expected_version="0.18.0", process_exit_code=0, ) @@ -560,11 +560,11 @@ def test_report_types_are_strict_and_unknown_fields_are_dropped(self): parsed, producer = _parse_component_report( json.dumps(additive), expected_mode="preview", - expected_version="0.17.0", + expected_version="0.18.0", process_exit_code=0, ) self.assertNotIn("private_future_field", parsed) - self.assertEqual(producer, {"name": "provenance", "version": "0.17.0"}) + self.assertEqual(producer, {"name": "provenance", "version": "0.18.0"}) def test_nested_component_data_is_rebuilt_as_aggregate_only(self): report = component_report(status="blocked", exit_code=1) @@ -612,7 +612,7 @@ def test_nested_component_data_is_rebuilt_as_aggregate_only(self): parsed, _ = _parse_component_report( json.dumps(report), expected_mode="preview", - expected_version="0.17.0", + expected_version="0.18.0", process_exit_code=1, ) @@ -701,7 +701,7 @@ def test_required_nested_semantics_fail_closed(self): _parse_component_report( json.dumps(report), expected_mode="preview", - expected_version="0.17.0", + expected_version="0.18.0", process_exit_code=report["exit_code"], ) diff --git a/tests/test_resume.py b/tests/test_resume.py new file mode 100644 index 0000000..74bfdb2 --- /dev/null +++ b/tests/test_resume.py @@ -0,0 +1,277 @@ +from __future__ import annotations + +import contextlib +import io +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from scriptorium import cli +from scriptorium.config import SuiteConfig, render_config +from scriptorium.resume import ResumeError, _parse_capsule, format_resume_report, run_resume + + +def capsule() -> dict[str, object]: + return { + "capsule_version": "context-capsule/0.1", + "project": { + "project_id": "synthetic-catalyst", + "title": "Synthetic Catalyst Study", + "status": "active", + "stage": "evidence-review", + "priority": "high", + "updated": "2026-07-22", + "goal": "Test one falsifiable catalyst hypothesis.", + "conclusion": "", + }, + "next_actions": ["Compare the two synthetic papers."], + "blocked_by": "", + "recent_progress": [ + {"date": "2026-07-22", "items": ["Approved the research question."]} + ], + "literature": [ + { + "id": "paper-1", + "citekey": "synthetic2026", + "title": "A Synthetic Catalyst Paper", + "year": 2026, + "read_status": "deep_read", + "tldr": "A synthetic result for contract testing.", + } + ], + "research_artifacts": [ + { + "kind": "reading-note", + "schema_version": "reading-note/1.0", + "id": "reading-note-1", + "title": "Synthetic reading note", + "status": "reviewed", + "source_ids": ["paper-1"], + "summary": "Reference-only synthetic evidence.", + "trust": "reference_only", + } + ], + "reference_leads": { + "gaps": ["No synthetic replication yet."], + "priority_reads": ["paper-1"], + }, + "trust": { + "project_state": "human_or_approved", + "recent_progress": "auto_applied_low_risk_not_approved_claims", + "research_artifacts": "reference_only_not_approved_claims", + }, + "limits": {"max_markdown_chars": 8000, "truncated": False}, + } + + +def completed(value: dict[str, object], *, stderr: str = "") -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess( + args=["prov-context"], + returncode=0, + stdout=json.dumps(value), + stderr=stderr, + ) + + +class ResumeBoundaryTests(unittest.TestCase): + def test_run_resume_uses_public_command_and_suppresses_local_paths(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + home = root / "private data" + source = root / "Provenance" + home.mkdir() + source.mkdir() + with ( + mock.patch("scriptorium.resume._compatibility_version", return_value="0.17.0"), + mock.patch( + "scriptorium.resume._resolve_provenance", + return_value=(source, Path("prov-context")), + ), + mock.patch( + "scriptorium.resume.subprocess.run", + side_effect=[ + subprocess.CompletedProcess( + args=["prov-context", "--version"], + returncode=0, + stdout="0.17.0\n", + stderr="", + ), + completed(capsule(), stderr="private diagnostic"), + ], + ) as invoke, + ): + report = run_resume( + provenance_home=home, + project="synthetic-catalyst", + ) + + command = invoke.call_args_list[1].args[0] + self.assertEqual( + command, ["prov-context", "--project", "synthetic-catalyst", "--json"] + ) + self.assertEqual( + invoke.call_args_list[1].kwargs["env"]["PROVENANCE_HOME"], + str(home.resolve()), + ) + self.assertEqual(report["operation"], "resume") + self.assertEqual(report["capsule"]["project"]["project_id"], "synthetic-catalyst") + self.assertEqual(report["entry"]["stderr"], "suppressed") + rendered = json.dumps(report) + self.assertNotIn(str(home), rendered) + self.assertNotIn("private diagnostic", rendered) + + def test_unknown_component_fields_fail_closed(self): + value = capsule() + value["private_path"] = "hidden" + with self.assertRaisesRegex(ResumeError, "root.*incompatible"): + _parse_capsule(json.dumps(value)) + + def test_local_paths_fail_closed(self): + value = capsule() + value["project"]["goal"] = r"Read C:\Users\Researcher\private.md" + with self.assertRaisesRegex(ResumeError, "suppressed local path"): + _parse_capsule(json.dumps(value)) + + def test_reference_artifacts_cannot_claim_authoritative_trust(self): + value = capsule() + value["research_artifacts"][0]["trust"] = "authoritative" + with self.assertRaisesRegex(ResumeError, "artifact trust"): + _parse_capsule(json.dumps(value)) + + def test_component_failure_does_not_forward_diagnostics(self): + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary) + failed = subprocess.CompletedProcess( + args=["prov-context"], + returncode=2, + stdout="", + stderr=r"unknown project under C:\Users\Researcher\private", + ) + with ( + mock.patch("scriptorium.resume._compatibility_version", return_value="0.17.0"), + mock.patch( + "scriptorium.resume._resolve_provenance", + return_value=(None, Path("prov-context")), + ), + mock.patch( + "scriptorium.resume.subprocess.run", + side_effect=[ + subprocess.CompletedProcess( + args=["prov-context", "--version"], + returncode=0, + stdout="0.17.0\n", + stderr="", + ), + failed, + ], + ), + ): + with self.assertRaisesRegex(ResumeError, "did not return a capsule") as raised: + run_resume(provenance_home=home, project="missing") + self.assertNotIn("Researcher", str(raised.exception)) + + def test_runtime_version_mismatch_fails_before_capsule_request(self): + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary) + with ( + mock.patch( + "scriptorium.resume._compatibility_version", + return_value="0.17.0", + ), + mock.patch( + "scriptorium.resume._resolve_provenance", + return_value=(None, Path("prov-context")), + ), + mock.patch( + "scriptorium.resume.subprocess.run", + return_value=subprocess.CompletedProcess( + args=["prov-context", "--version"], + returncode=0, + stdout="0.16.0\n", + stderr=r"private C:\Users\Researcher", + ), + ) as invoke, + ): + with self.assertRaisesRegex( + ResumeError, "incompatible runtime version" + ) as raised: + run_resume(provenance_home=home, project="synthetic-catalyst") + + self.assertEqual(invoke.call_count, 1) + self.assertNotIn("Researcher", str(raised.exception)) + + def test_human_report_marks_research_artifacts_reference_only(self): + report = { + "generated_by": {"version": "0.1.0"}, + "capsule": capsule(), + "path_selection": {}, + "warnings": [], + } + rendered = format_resume_report(report) + self.assertIn("Synthetic Catalyst Study", rendered) + self.assertIn("Reference context", rendered) + self.assertIn("reference-only", rendered) + + +class ResumeCliTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.workspace = self.root / "workspace" + self.home = self.root / "provenance" + self.workspace.mkdir() + self.home.mkdir() + self.config_root = self.root / "config" + config_path = self.config_root / "scriptorium" / "config.toml" + config_path.parent.mkdir(parents=True) + config_path.write_bytes( + render_config( + SuiteConfig( + workspace=self.workspace.resolve(), + provenance_home=self.home.resolve(), + hosts=("codex",), + default_project="configured-project", + ) + ) + ) + + def invoke_json(self, arguments: list[str]) -> tuple[int, dict[str, object], str]: + stdout, stderr = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + exit_code = cli.main(arguments) + return exit_code, json.loads(stdout.getvalue()), stderr.getvalue() + + def test_resume_uses_configured_home_and_project(self): + component_report = {"operation": "resume", "exit_code": 0} + with ( + mock.patch.dict(os.environ, {}, clear=True), + mock.patch("scriptorium.cli.run_resume", return_value=component_report) as run, + ): + exit_code, report, stderr = self.invoke_json( + ["resume", "--config-dir", str(self.config_root), "--json"] + ) + + self.assertEqual(exit_code, 0) + self.assertEqual(stderr, "") + self.assertEqual(run.call_args.kwargs["provenance_home"], self.home.resolve()) + self.assertEqual(run.call_args.kwargs["project"], "configured-project") + self.assertEqual(report["path_selection"]["data_root"]["source"], "suite-config") + + def test_resume_json_usage_error_never_echoes_private_arguments(self): + secret = r"C:\Users\Researcher\private-project" + exit_code, report, stderr = self.invoke_json( + ["resume", "--project", secret, "--json", "--unknown", secret] + ) + self.assertEqual(exit_code, 2) + self.assertEqual(stderr, "") + self.assertEqual(report["operation"], "resume") + self.assertNotIn(secret, json.dumps(report)) + + +if __name__ == "__main__": + unittest.main() diff --git a/uv.lock b/uv.lock index 5ff22cb..c3a1af6 100644 --- a/uv.lock +++ b/uv.lock @@ -4,5 +4,5 @@ requires-python = ">=3.11" [[package]] name = "scriptorium-suite" -version = "0.1.0" +version = "0.2.0" source = { editable = "." }