From eb1c05056b27a30a7869bb75f31c1f11aae2d034 Mon Sep 17 00:00:00 2001 From: dytlzl Date: Wed, 18 Nov 2026 15:18:16 +0900 Subject: [PATCH] feat: show verified evidence in HTML and run Playwright checks --- .github/workflows/check.yml | 28 +++ .pdd/reviews/httpwallet-browser.json | 149 +++++++++++ README.md | 6 +- docs/browser-checks.md | 63 +++++ docs/cli-json.md | 5 + docs/design.md | 2 +- docs/plan-format.md | 15 +- docs/reference-comparison.md | 7 +- examples/httpwallet/README.md | 11 + examples/httpwallet/browser/plan.json | 125 ++++++++++ .../httpwallet/browser/playwright.config.cjs | 18 ++ examples/httpwallet/browser/wallet.spec.cjs | 14 ++ examples/httpwallet/cmd/wallet/main.go | 15 +- examples/httpwallet/cmd/wallet/wallet.html | 52 ++++ internal/cli/cli.go | 2 +- internal/pdd/browser.go | 232 ++++++++++++++++++ internal/pdd/browser_test.go | 128 ++++++++++ internal/pdd/dashboard.go | 82 ++++++- internal/pdd/dashboard.html | 132 ++++++++++ internal/pdd/inputs.go | 10 + internal/pdd/plan.go | 17 +- internal/pdd/planning.go | 2 +- internal/pdd/prove.go | 75 +++++- internal/pdd/runner.go | 7 +- internal/pdd/workflow.go | 5 +- 25 files changed, 1164 insertions(+), 38 deletions(-) create mode 100644 .pdd/reviews/httpwallet-browser.json create mode 100644 docs/browser-checks.md create mode 100644 examples/httpwallet/browser/plan.json create mode 100644 examples/httpwallet/browser/playwright.config.cjs create mode 100644 examples/httpwallet/browser/wallet.spec.cjs create mode 100644 examples/httpwallet/cmd/wallet/wallet.html create mode 100644 internal/pdd/browser.go create mode 100644 internal/pdd/browser_test.go create mode 100644 internal/pdd/dashboard.html diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index fa22991..a847525 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -75,6 +75,32 @@ jobs: assert result['data']['current'] and result['data']['receipt']['success'] assert result['data']['receipt']['approval']['reviewer'] == 'codex-example-review' PYTHON + - name: Install the browser test runner outside the source snapshot + run: | + npm install --no-audit --no-fund --prefix "$RUNNER_TEMP/pdd-playwright" @playwright/test@1.65.1 + runner="$RUNNER_TEMP/pdd-playwright/node_modules/@playwright/test/cli.js" + node "$runner" install --with-deps chromium + echo "PDD_PLAYWRIGHT_CLI=$runner" >> "$GITHUB_ENV" + - name: Verify the wallet browser flow and HTML evidence + run: | + bin/pdd --plan httpwallet-browser init --from examples/httpwallet/browser/plan.json + bin/pdd --plan httpwallet-browser import-approval --from .pdd/reviews/httpwallet-browser.json + bin/pdd --plan httpwallet-browser prove --json > .pdd/browser-prove-result.json + bin/pdd --plan httpwallet-browser status --json > .pdd/browser-status-result.json + bin/pdd --plan httpwallet-browser report --out .pdd/state/httpwallet-browser/report.html + python3 - <<'PYCODE' + import json + from pathlib import Path + status = json.loads(Path('.pdd/browser-status-result.json').read_text()) + assert status['ok'] and status['data']['current'] + receipt = status['data']['receipt'] + assert receipt['success'] + assert any(v['id'] == 'overdraft-screen' and v['status'] == 'evidenced' for v in receipt['requirements']) + browser = next(e for e in receipt['evidence'] if e['check'] == 'wallet-browser') + assert browser['kind'] == 'empirical' and browser['artifacts'][0]['file'] == 'wallet-browser.playwright.json' + report = Path('.pdd/state/httpwallet-browser/report.html').read_text() + assert 'playwright · empirical' in report and 'wallet-browser.playwright.json' in report and 'killed' in report + PYCODE - name: Upload proof receipts if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 @@ -83,6 +109,8 @@ jobs: path: | .pdd/receipts/ .pdd/state/httpwallet/receipts/ + .pdd/state/httpwallet-browser/receipts/ + .pdd/state/httpwallet-browser/report.html .pdd/*-result.json if-no-files-found: warn include-hidden-files: true diff --git a/.pdd/reviews/httpwallet-browser.json b/.pdd/reviews/httpwallet-browser.json new file mode 100644 index 0000000..d0a3553 --- /dev/null +++ b/.pdd/reviews/httpwallet-browser.json @@ -0,0 +1,149 @@ +{ + "schema": 1, + "digest": "de201c3d481e039c7c435f4ce41734a4e25039a6c105e3fdd0b01abca4ebe699", + "reviewer": "codex-example-review", + "accepted_trust": [ + "wallet-runtime", + "wallet-sessions" + ], + "approved_at": "2026-11-18T06:14:12.837703671Z", + "plan_id": "httpwallet-browser", + "review": { + "schema": 1, + "plan_id": "httpwallet-browser", + "plan": { + "scenarios": [ + { + "id": "browser-overdraft", + "check": "wallet-browser", + "test": "wallet.spec.cjs :: rejects an overdraft without debiting the wallet", + "given": "A demo account starts with a balance of 100", + "when": "The user attempts to withdraw 101 then withdraws 100", + "then": "The browser shows a denial followed by a successful withdrawal with zero remaining", + "requirements": [ + "overdraft-screen" + ], + "purpose": "integration" + } + ], + "schema": 1, + "name": "HTTP wallet with browser evidence", + "intent": "The HTTP withdrawal endpoint must use the verified production decision, allowing a withdrawal exactly for an authenticated, unlocked account with a positive amount within its balance. Exercise request decoding, sessions and atomic state updates empirically. Show a refusal in the browser for an overdraw and leave the wallet unchanged.", + "non_goals": [ + "The static demo tokens are not production authentication; no TLS, identity proof, persistence, payment settlement or distributed consistency is provided.", + "Gobra proves the pure decision only; HTTP decoding, the session mapping, synchronization and state updates have empirical evidence, not formal proof.", + "Availability, fairness, transport failure recovery and exactly-once request semantics are outside scope." + ], + "requirements": [ + { + "id": "withdrawal-policy", + "text": "Accept exactly authenticated, unlocked withdrawals with positive amount no greater than balance.", + "kind": "formal", + "check": "withdrawal-proof", + "file": "examples/httpwallet/policy/policy.go", + "function": "CanWithdraw", + "ensures": "allowed == (authenticated \u0026\u0026 !locked \u0026\u0026 amount \u003e 0 \u0026\u0026 amount \u003c= balance)" + }, + { + "id": "http-boundary", + "text": "HTTP method, path, session and withdrawal boundaries return expected status and balance, without changing funds on rejection.", + "kind": "empirical", + "check": "http-tests" + }, + { + "id": "http-decoding", + "text": "Malformed, oversized, ambiguous and unknown request inputs are rejected without changing funds.", + "kind": "empirical", + "check": "http-tests" + }, + { + "id": "atomic-withdrawals", + "text": "Concurrent requests through a real local HTTP server spend the initial balance exactly once without overdraft.", + "kind": "empirical", + "check": "http-tests" + }, + { + "id": "overdraft-screen", + "text": "A browser shows withdrawal denied for an overdraft and the full starting balance remains available.", + "kind": "empirical", + "check": "wallet-browser" + } + ], + "checks": [ + { + "id": "withdrawal-proof", + "kind": "gobra", + "files": [ + "examples/httpwallet/policy/policy.go" + ] + }, + { + "id": "http-tests", + "kind": "go-test", + "package": "./examples/httpwallet/app", + "tests": [ + "TestWithdrawalHTTP", + "TestMalformedHTTP", + "TestConcurrentHTTPWithdrawals" + ] + }, + { + "id": "wallet-browser", + "kind": "playwright", + "tests": [ + "wallet.spec.cjs :: rejects an overdraft without debiting the wallet" + ], + "config": "examples/httpwallet/browser/playwright.config.cjs", + "project": "chromium" + } + ], + "trusted_boundary": [ + { + "id": "wallet-sessions", + "assumption": "The configured session mapping identifies the correct account and lock state; demo tokens are public test credentials. Tests exercise known, unknown, locked and ambiguous tokens but cannot establish a real identity provider's truthfulness.", + "check": "http-tests" + }, + { + "id": "wallet-runtime", + "assumption": "Go integer/JSON/HTTP implementations and mutex semantics are trusted. Race and TCP concurrency tests exercise parsing and an atomic in-memory update; they do not prove the runtime or provide durable financial guarantees.", + "check": "http-tests" + } + ], + "mutations": [ + { + "id": "ignore-session-lock", + "requirement": "withdrawal-policy", + "check": "withdrawal-proof", + "file": "examples/httpwallet/policy/policy.go", + "find": "return authenticated \u0026\u0026 !locked \u0026\u0026 amount \u003e 0 \u0026\u0026 amount \u003c= balance", + "replace": "return authenticated \u0026\u0026 amount \u003e 0 \u0026\u0026 amount \u003c= balance" + }, + { + "id": "allow-zero-withdrawal", + "requirement": "withdrawal-policy", + "check": "withdrawal-proof", + "file": "examples/httpwallet/policy/policy.go", + "find": "return authenticated \u0026\u0026 !locked \u0026\u0026 amount \u003e 0 \u0026\u0026 amount \u003c= balance", + "replace": "return authenticated \u0026\u0026 !locked \u0026\u0026 amount \u003e= 0 \u0026\u0026 amount \u003c= balance" + }, + { + "id": "allow-overdraft", + "requirement": "withdrawal-policy", + "check": "withdrawal-proof", + "file": "examples/httpwallet/policy/policy.go", + "find": "return authenticated \u0026\u0026 !locked \u0026\u0026 amount \u003e 0 \u0026\u0026 amount \u003c= balance", + "replace": "return authenticated \u0026\u0026 !locked \u0026\u0026 amount \u003e 0" + } + ] + }, + "sources": [ + { + "file": "examples/httpwallet/policy/policy.go", + "annotations": [ + "ensures allowed == (authenticated \u0026\u0026 !locked \u0026\u0026 amount \u003e 0 \u0026\u0026 amount \u003c= balance)", + "declarations:package policy\n\nfunc CanWithdraw(authenticated, locked bool, amount, balance int) (allowed bool) {\n}\n" + ] + } + ] + } +} diff --git a/README.md b/README.md index 82ebb61..8d2c9e9 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ login-decision: proved (formal) login-boundary: evidenced (empirical) ``` -Open the `receipt.md` path printed by `pdd prove` to see the checks, rejected mutations, trusted assumptions, and proof limits. `pdd status` checks that the latest receipt still matches the current source and approval. For a browser view of the plan and evidence, run `pdd report --out .pdd/results/login.html` and open that file. +Open the `receipt.md` path printed by `pdd prove` to see the checks, rejected mutations, trusted assumptions, and proof limits. `pdd status` checks that the latest receipt still matches the current source and approval. For the requirements, test results, rejected mutations, and links to logs on one page, run `pdd report --out .pdd/results/login.html` and open the HTML file. ## Use it in your project @@ -64,14 +64,14 @@ If you already work with a coding agent, `pdd draft` can turn a request into an | Example | What to look at | | --- | --- | -| [HTTP wallet](examples/httpwallet/README.md) | A real HTTP handler calls a proved withdrawal rule. Go tests exercise session handling, invalid requests, and concurrent withdrawals. | +| [HTTP wallet](examples/httpwallet/README.md) | A real HTTP handler calls a proved withdrawal rule. Go tests exercise the HTTP boundary and concurrent withdrawals. The optional browser check submits an overdraft and confirms the balance remains available. | | [Multi-package authorization](examples/authorization/plan.json) | A handler calls local policy methods. PDD checks the dependency package before the caller. | | [Slice validation](examples/validation/README.md) | A production loop proves a property over every element using permissions, a quantified contract, and loop invariants. | ## What you can rely on - **Formal proofs cover stated contracts.** Gobra checks supported Go functions and methods, including supported calls across packages in one module. External and standard-library imports in proof packages remain outside the supported scope. [Supported proof inputs](docs/plan-format.md) -- **Go tests provide evidence for the cases they run.** The receipt labels them separately from proofs. Each planned negative mutation must fail on the intended postcondition. A missing test, tool failure, or inconclusive mutation fails verification. [Design](docs/design.md) +- **Go and browser tests provide evidence for the cases they run.** The receipt labels them separately from proofs. Each planned negative mutation must fail on the intended postcondition. A missing test, tool failure, or inconclusive mutation fails verification. [Design](docs/design.md) - **Receipts tie the result to the reviewed change.** Machine-readable JSON and Markdown include the plan and source digests, commands, outcomes, trust assumptions, and links or references to saved logs. `status` rejects stale evidence. Use Ed25519 approvals with a trusted policy when CI needs to verify reviewer authority. [JSON protocol](docs/cli-json.md) · [Signed reviews](docs/signed-reviews.md) `pdd-kit-go` is an independent implementation of ideas from the Proof-Driven Development Kit research prototype. Its Go verifier is Gobra. [Original PDD Kit](https://githubnext.com/projects/pdd-kit/) · [Feature comparison and limits](docs/reference-comparison.md) diff --git a/docs/browser-checks.md b/docs/browser-checks.md new file mode 100644 index 0000000..42d8301 --- /dev/null +++ b/docs/browser-checks.md @@ -0,0 +1,63 @@ +# Browser checks with Playwright + +A `playwright` check runs an existing Playwright Test project in the same kind of +isolated source copy as a Go test. PDD verifies the required test identities in +the final JSON report and saves it with the receipt. HTML reports link to the +execution log, the Playwright JSON, and any saved failure screenshots and traces. + +## Run the wallet example + +Use a Go toolchain, the pinned Gobra image, and Node.js 20 or newer for this +example. Install Playwright and Chromium once on the machine running the check. +Run these commands from the repository root with a `pdd` binary built from this +checkout. + +```sh +browser_tools=$(mktemp -d) +npm install --prefix "$browser_tools" @playwright/test@1.65.1 +export PDD_PLAYWRIGHT_CLI="$browser_tools/node_modules/@playwright/test/cli.js" +node "$PDD_PLAYWRIGHT_CLI" install chromium + +pdd --plan httpwallet-browser init --from examples/httpwallet/browser/plan.json +pdd --plan httpwallet-browser plan +pdd --plan httpwallet-browser import-approval --from .pdd/reviews/httpwallet-browser.json +pdd --plan httpwallet-browser prove +pdd --plan httpwallet-browser status +pdd --plan httpwallet-browser report --out .pdd/state/httpwallet-browser/report.html +``` + +Open the generated HTML alongside its `receipts/` directory so relative links +work. The saved example approval is a demonstration assertion. Review and approve +your own project's requirements and trust assumptions separately. On Linux, +`node "$PDD_PLAYWRIGHT_CLI" install --with-deps chromium` can also install the +browser's system dependencies. + +## Adopt an existing suite + +- Point `PDD_PLAYWRIGHT_CLI` to an absolute `node_modules/@playwright/test/cli.js` + path. `node` must be on `PATH`. Provision the runner and any CommonJS test + dependencies together. PDD sets `NODE_PATH` to that `node_modules` directory. + The check performs no dependency installation. +- Add a check of kind `playwright` with `config`, `project`, and `tests` in the + [plan format](plan-format.md). Use a standalone Playwright project and a + repository-relative configuration. PDD runs that project without project + dependencies and verifies every required test against the completed result. + Use a config that starts its own local app and keeps output out of the source + snapshot. The wallet config sets `reuseExistingServer: false`. +- Keep project code, config, and tests in the captured snapshot. You can put the + preinstalled dependencies outside the repository, or explicitly exclude + `node_modules` via the reviewed `inputs.exclude` policy. Browser plans allow + only `node_modules` exclusions in addition to PDD's default reserved paths. + Use CommonJS-compatible dependency resolution for externally provisioned + packages. Fully native ESM imports do not use `NODE_PATH`. +- Set `use.screenshot` to `only-on-failure` if you want failure images. PDD + requests tracing on failure and retains bounded PNG/trace artifacts from + Playwright's output directory. Open a saved trace with + `node "$PDD_PLAYWRIGHT_CLI" show-trace PATH_TO_TRACE.zip`. Browser artifacts can + contain private page content or network data. Share them with the same care as + test logs. + +The runtime and browser are part of the empirical test environment. Playwright's +configuration and tests run with the invoking user's permissions, as Go tests do. +Browser checks provide evidence for the cases that actually ran. Gobra continues +to prove the Go contracts separately. diff --git a/docs/cli-json.md b/docs/cli-json.md index 4237d1f..bd1a299 100644 --- a/docs/cli-json.md +++ b/docs/cli-json.md @@ -80,3 +80,8 @@ and conservative pruning candidates. `build` returns stage, success, decision-ne state, applied edits, iterations and artifact paths; success=false exits 1. `report` returns its written HTML path. `pr` returns the evidence body/path and a URL only after explicit successful `--create`. All use the same schema-1 envelope. + +Playwright evidence can include an `artifacts` array on a receipt check. Each +entry has a receipt-relative `file` and `sha256`. The JSON test report is required +for a successful browser check. Optional PNG and trace ZIP files preserve failure +context. `status` revalidates referenced files and the required test results. diff --git a/docs/design.md b/docs/design.md index 7fae64c..3e43f55 100644 --- a/docs/design.md +++ b/docs/design.md @@ -20,7 +20,7 @@ The selected image is Gobra `v26.09`, resolved and pinned to `sha256:5e96431b06c - `packages.go` resolves complete same-module proof packages, receiver-qualified symbols and the dependency DAG. It rejects ambiguous import resolution and unverified callee packages. - `inputs.go` validates reviewed asset exclusions and the protected Go input closure; `snapshot.go` captures bounded inputs, modes and directories and creates separate check/mutation workspaces. - `approval_transfer.go` exports/imports complete matching reviews without minting CI approvals. -- `runner.go` invokes only Gobra and named Go tests, distinguishes operational failures and validates actual results. +- `runner.go` invokes Gobra and named Go tests. `browser.go` runs a reviewed Playwright project and validates first-attempt results for the named tests. - `prove.go` joins the evidence, rejects gaps and staleness, and writes receipts. Formal evidence covers exactly the clauses checked by the Gobra invocation. The production function is both compiled by Go and passed directly to Gobra. The baseline check succeeds first; each mutant must then fail on the target clause. No successful test can replace a failed formal obligation. diff --git a/docs/plan-format.md b/docs/plan-format.md index 3107fc2..5f34129 100644 --- a/docs/plan-format.md +++ b/docs/plan-format.md @@ -10,7 +10,7 @@ The [login example](../examples/login/plan.json) is a complete plan. Identifiers | `name`, `intent` | Human-readable purpose and intended behavior | | `non_goals` | Limitations and excluded guarantees shown in receipts | | `requirements` | Named obligations with a formal or empirical evidence classification | -| `checks` | Typed Gobra or Go-test invocations; arbitrary shell commands are not supported | +| `checks` | Typed Gobra, Go test, or Playwright invocations | | `trusted_boundary` | Explicit assumptions with empirical checks | | `mutations` | Negative examples, at least one per formal requirement | | `inputs` | Optional reviewed snapshot limits and exact asset exclusions; see [input scope](inputs.md) | @@ -27,7 +27,18 @@ External/standard-library imports, nested modules, `go.mod` replacement directiv An empirical requirement names `id`, `text`, `kind: "empirical"`, and a `go-test` check. That check has `package` (one local package, such as `./internal/auth`) and `tests` (explicit top-level names such as `TestLoginBoundary`). Passing output must contain an actual `pass` event for every named test. Table cases/subtests may exist inside those named tests. Test quality and sufficiency require review; a passing test can still assert nothing useful. -Each trust item has `id`, `assumption`, and `check`; the check must be empirical. `approve --accept-tcb` must list exactly all trust IDs. Mapping a broad assumption to one weak test does not make it true: explain the test's limits in the assumption and non-goals. +A `playwright` check supplies `config` (repository-relative config file), `project` +(one exact project name), and `tests` (required test identities). A test identity +has the form `file :: title`, with the file relative to Playwright's report root. +For nested `describe` blocks, use `file :: Suite › Nested suite › test title`. +Use this check for empirical requirements, trust checks, and test scenarios. + +The selected project runs once, with zero retries. Every listed test must appear +as a first-attempt pass in the final Playwright JSON, and the project must finish +without skipped, failed, flaky, or expected-failure cases. The results are recorded +as empirical evidence. See [browser checks](browser-checks.md) for setup and scope. + +Each trust item has `id`, `assumption`, and `check`; the check must be empirical. `approve --accept-tcb` must list all trust IDs and assumed clause IDs. Mapping a broad assumption to one weak test does not make it true: explain the test's limits in the assumption and non-goals. ## Negative mutations diff --git a/docs/reference-comparison.md b/docs/reference-comparison.md index 5eb0e2b..17dd249 100644 --- a/docs/reference-comparison.md +++ b/docs/reference-comparison.md @@ -16,9 +16,9 @@ planning. These published workflows are the basis for the comparison below. | Intent artifact | Original request, verbatim excerpts mapped to requirement/decision IDs | Human review determines whether the proposed clauses express the right intent | | Planning review | Two separate invocations for intent fidelity and guarantee quality; both must be ready before approval when enabled | Advisory opinions, not authenticated independent people or formal proof | | Trust | Per-clause assumptions, impact if false, named tests, Given/When/Then cases and explicit acceptance of each clause | Tests sample behavior; external library implementations remain outside Gobra's proved core | -| Human-facing views | Offline HTML cards, specification landscape, workflow diagram, task dependencies, findings and limits | No Chopin API integration, live multiplayer editor or generated semantic pictures | +| Human-facing views | Offline HTML for requirements, verdicts, negative examples, trust, review and linked artifacts | No Chopin API integration, live multiplayer editor or generated semantic pictures | | Implementation | Scratch candidate repair loop for Go bodies, tests and opt-in provable proof hints | Frozen declarations, contracts, pure definitions, intent and trust cannot be silently changed | -| Verification | Pinned Gobra proofs, named Go tests, per-clause negative mutations, source/approval/log-bound receipts | Supported Go subset; not Rust/Verus equivalence | +| Verification | Pinned Gobra proofs, named Go and Playwright tests, per-clause negative mutations and source-bound receipts | Supported Go subset; not Rust/Verus equivalence | | Gap analysis | Separate advisory reviewer sees source and actual machine results; review binds to the exact receipt | A model cannot turn failed machine verification into success | | Feedback | Machine/advisory failures lead to bounded repair; changed contracts or unsupported scope stop for a decision | No automatic human signoff | | Delivery | `pr` previews an evidence-backed body; `pr --create` opens a draft PR for committed, pushed, freshly verified code | Does not push, merge or bypass protection | @@ -26,8 +26,7 @@ planning. These published workflows are the basis for the comparison below. The remaining reference-specific gaps are real: Chopin's collaborative planning surface, Rust async and Verus-specific trusted external function adapters, and -built-in browser/computer-use test orchestration are not implemented. Go empirical -tests can exercise real I/O; that does not prove those dependencies. Arbitrary Go +general desktop computer-use orchestration remain open. The built-in Playwright adapter runs reviewed browser suites and records empirical results. Arbitrary Go generics, dependency imports in formal packages and concurrency proofs retain the restrictions documented in [the plan format](plan-format.md). diff --git a/examples/httpwallet/README.md b/examples/httpwallet/README.md index e2d240c..381563e 100644 --- a/examples/httpwallet/README.md +++ b/examples/httpwallet/README.md @@ -10,6 +10,7 @@ curl -i http://127.0.0.1:8080/v1/withdraw \ -H 'Content-Type: application/json' -d '{"amount":40}' ``` +Open `http://127.0.0.1:8080/` for the demo form, or use the API request above. The response is `200` with `{"balance":60}`. The in-memory balance starts at 100; a second request for 100 returns `403`. `demo-locked` is always denied. Restarting resets all balances. These public static credentials, single-process memory and @@ -45,3 +46,13 @@ for HTTP, session lookup and atomic updates. Client disconnects may occur after commit; retries are not idempotent. No durable storage, distributed locking, identity verification, TLS or settlement guarantees are claimed. The reviewable plan records these limits and the runtime/session trust boundary explicitly. + +## Browser verification + +The separate `httpwallet-browser` plan proves the same Go policy, runs the HTTP +and concurrency tests, and opens this form in Chromium. The browser case attempts +to withdraw 101 from a balance of 100, sees the denial, then withdraws all 100. +The final response displays a remaining balance of zero. + +Follow the [browser setup and example commands](../../docs/browser-checks.md). +The example plan uses port 38479 and starts a fresh server for the check. diff --git a/examples/httpwallet/browser/plan.json b/examples/httpwallet/browser/plan.json new file mode 100644 index 0000000..471c0c2 --- /dev/null +++ b/examples/httpwallet/browser/plan.json @@ -0,0 +1,125 @@ +{ + "schema": 1, + "name": "HTTP wallet with browser evidence", + "intent": "The HTTP withdrawal endpoint must use the verified production decision, allowing a withdrawal exactly for an authenticated, unlocked account with a positive amount within its balance. Exercise request decoding, sessions and atomic state updates empirically. Show a refusal in the browser for an overdraw and leave the wallet unchanged.", + "non_goals": [ + "The static demo tokens are not production authentication; no TLS, identity proof, persistence, payment settlement or distributed consistency is provided.", + "Gobra proves the pure decision only; HTTP decoding, the session mapping, synchronization and state updates have empirical evidence, not formal proof.", + "Availability, fairness, transport failure recovery and exactly-once request semantics are outside scope." + ], + "requirements": [ + { + "id": "withdrawal-policy", + "text": "Accept exactly authenticated, unlocked withdrawals with positive amount no greater than balance.", + "kind": "formal", + "check": "withdrawal-proof", + "file": "examples/httpwallet/policy/policy.go", + "function": "CanWithdraw", + "ensures": "allowed == (authenticated && !locked && amount > 0 && amount <= balance)" + }, + { + "id": "http-boundary", + "text": "HTTP method, path, session and withdrawal boundaries return expected status and balance, without changing funds on rejection.", + "kind": "empirical", + "check": "http-tests" + }, + { + "id": "http-decoding", + "text": "Malformed, oversized, ambiguous and unknown request inputs are rejected without changing funds.", + "kind": "empirical", + "check": "http-tests" + }, + { + "id": "atomic-withdrawals", + "text": "Concurrent requests through a real local HTTP server spend the initial balance exactly once without overdraft.", + "kind": "empirical", + "check": "http-tests" + }, + { + "id": "overdraft-screen", + "text": "A browser shows withdrawal denied for an overdraft and the full starting balance remains available.", + "kind": "empirical", + "check": "wallet-browser" + } + ], + "checks": [ + { + "id": "withdrawal-proof", + "kind": "gobra", + "files": [ + "examples/httpwallet/policy/policy.go" + ] + }, + { + "id": "http-tests", + "kind": "go-test", + "package": "./examples/httpwallet/app", + "tests": [ + "TestWithdrawalHTTP", + "TestMalformedHTTP", + "TestConcurrentHTTPWithdrawals" + ] + }, + { + "id": "wallet-browser", + "kind": "playwright", + "config": "examples/httpwallet/browser/playwright.config.cjs", + "project": "chromium", + "tests": [ + "wallet.spec.cjs :: rejects an overdraft without debiting the wallet" + ] + } + ], + "trusted_boundary": [ + { + "id": "wallet-sessions", + "assumption": "The configured session mapping identifies the correct account and lock state; demo tokens are public test credentials. Tests exercise known, unknown, locked and ambiguous tokens but cannot establish a real identity provider's truthfulness.", + "check": "http-tests" + }, + { + "id": "wallet-runtime", + "assumption": "Go integer/JSON/HTTP implementations and mutex semantics are trusted. Race and TCP concurrency tests exercise parsing and an atomic in-memory update; they do not prove the runtime or provide durable financial guarantees.", + "check": "http-tests" + } + ], + "mutations": [ + { + "id": "ignore-session-lock", + "requirement": "withdrawal-policy", + "check": "withdrawal-proof", + "file": "examples/httpwallet/policy/policy.go", + "find": "return authenticated && !locked && amount > 0 && amount <= balance", + "replace": "return authenticated && amount > 0 && amount <= balance" + }, + { + "id": "allow-zero-withdrawal", + "requirement": "withdrawal-policy", + "check": "withdrawal-proof", + "file": "examples/httpwallet/policy/policy.go", + "find": "return authenticated && !locked && amount > 0 && amount <= balance", + "replace": "return authenticated && !locked && amount >= 0 && amount <= balance" + }, + { + "id": "allow-overdraft", + "requirement": "withdrawal-policy", + "check": "withdrawal-proof", + "file": "examples/httpwallet/policy/policy.go", + "find": "return authenticated && !locked && amount > 0 && amount <= balance", + "replace": "return authenticated && !locked && amount > 0" + } + ], + "scenarios": [ + { + "id": "browser-overdraft", + "check": "wallet-browser", + "test": "wallet.spec.cjs :: rejects an overdraft without debiting the wallet", + "given": "A demo account starts with a balance of 100", + "when": "The user attempts to withdraw 101 then withdraws 100", + "then": "The browser shows a denial followed by a successful withdrawal with zero remaining", + "requirements": [ + "overdraft-screen" + ], + "purpose": "integration" + } + ] +} diff --git a/examples/httpwallet/browser/playwright.config.cjs b/examples/httpwallet/browser/playwright.config.cjs new file mode 100644 index 0000000..042dfdc --- /dev/null +++ b/examples/httpwallet/browser/playwright.config.cjs @@ -0,0 +1,18 @@ +const path = require('node:path'); + +const origin = 'http://127.0.0.1:38479'; +module.exports = { + testDir: __dirname, + workers: 1, + projects: [{ + name: 'chromium', + use: { browserName: 'chromium', baseURL: origin, screenshot: 'only-on-failure' }, + }], + webServer: { + command: 'go run ./examples/httpwallet/cmd/wallet -listen 127.0.0.1:38479', + cwd: path.resolve(__dirname, '../../..'), + url: origin, + reuseExistingServer: false, + timeout: 120_000, + }, +}; diff --git a/examples/httpwallet/browser/wallet.spec.cjs b/examples/httpwallet/browser/wallet.spec.cjs new file mode 100644 index 0000000..7deb588 --- /dev/null +++ b/examples/httpwallet/browser/wallet.spec.cjs @@ -0,0 +1,14 @@ +const { test, expect } = require('@playwright/test'); + +test('rejects an overdraft without debiting the wallet', async ({ page }) => { + await page.goto('/'); + await page.getByLabel('Amount').fill('101'); + await page.getByRole('button', { name: 'Withdraw' }).click(); + await expect(page.getByRole('status')).toHaveText('Withdrawal denied.'); + + // Withdrawing the entire starting balance proves the rejected attempt did + // not debit it on this path. The success message uses the server's balance. + await page.getByLabel('Amount').fill('100'); + await page.getByRole('button', { name: 'Withdraw' }).click(); + await expect(page.getByRole('status')).toHaveText('Remaining balance: 0'); +}); diff --git a/examples/httpwallet/cmd/wallet/main.go b/examples/httpwallet/cmd/wallet/main.go index 3a76cee..ddda50b 100644 --- a/examples/httpwallet/cmd/wallet/main.go +++ b/examples/httpwallet/cmd/wallet/main.go @@ -4,6 +4,7 @@ package main import ( "context" + _ "embed" "flag" "log" "net/http" @@ -15,6 +16,9 @@ import ( "github.com/cruskell/pdd-kit-go/examples/httpwallet/app" ) +//go:embed wallet.html +var walletPage []byte + func main() { address := flag.String("listen", "127.0.0.1:8080", "HTTP listen address") flag.Parse() @@ -22,7 +26,16 @@ func main() { "demo-alice": {Account: "alice"}, "demo-locked": {Account: "locked", Locked: true}, }, map[string]int{"alice": 100, "locked": 100}) - server := &http.Server{Addr: *address, Handler: handler, ReadHeaderTimeout: 5 * time.Second, + demo := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/" && r.Method == http.MethodGet { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.Header().Set("Cache-Control", "no-store") + _, _ = w.Write(walletPage) + return + } + handler.ServeHTTP(w, r) + }) + server := &http.Server{Addr: *address, Handler: demo, ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 10 * time.Second, WriteTimeout: 10 * time.Second, IdleTimeout: 30 * time.Second, MaxHeaderBytes: 8192} ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() diff --git a/examples/httpwallet/cmd/wallet/wallet.html b/examples/httpwallet/cmd/wallet/wallet.html new file mode 100644 index 0000000..cff2f40 --- /dev/null +++ b/examples/httpwallet/cmd/wallet/wallet.html @@ -0,0 +1,52 @@ + + + + + + Demo wallet + + + +
+

Demo wallet

+
+ + + +
+

+
+ + + diff --git a/internal/cli/cli.go b/internal/cli/cli.go index da02e1a..c24725a 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -29,7 +29,7 @@ Usage: pdd [--cwd DIR] [--plan NAME] [--json] [options] approve Record review: --reviewer NAME [--accept-tcb id1,id2] export-approval Save the current reviewed assertion: --to PATH import-approval Restore a matching reviewed assertion: --from PATH - prove Run approved Gobra proofs, named Go tests and negative mutations + prove Run approved proofs, Go and Playwright checks, and negative mutations status Check whether this plan's latest receipt is current and successful doctor Check Go and Docker/Java prerequisites draft Generate a plan from --request-file; --apply installs an unapproved draft diff --git a/internal/pdd/browser.go b/internal/pdd/browser.go new file mode 100644 index 0000000..752c22a --- /dev/null +++ b/internal/pdd/browser.go @@ -0,0 +1,232 @@ +package pdd + +import ( + "context" + "encoding/json" + "fmt" + "io/fs" + "os" + "path" + "path/filepath" + "regexp" + "strings" +) + +const ( + browserArtifactLimit = 8 << 20 + browserArtifactCount = 8 +) + +var browserProjectRE = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]*$`) + +func isEmpirical(c Check) bool { return c.Kind == "go-test" || c.Kind == "playwright" } + +func browserRelativeFile(name string) bool { + return fs.ValidPath(name) && name != "." && !strings.ContainsAny(name, "\\:\r\n\x00") +} + +func validateBrowserCheck(root string, c Check) error { + if c.Package != "" || len(c.Files) != 0 || !browserRelativeFile(c.Config) || !browserProjectRE.MatchString(c.Project) || len(c.Tests) == 0 { + return fmt.Errorf("playwright needs a local config file, a single named project and explicit tests") + } + config, err := Local(root, c.Config) + if err != nil { + return err + } + st, err := os.Stat(config) + if err != nil || !st.Mode().IsRegular() { + return fmt.Errorf("missing Playwright config %s", c.Config) + } + seen := map[string]bool{} + for _, name := range c.Tests { + file, title, ok := strings.Cut(name, " :: ") + if !ok || !browserRelativeFile(file) || strings.TrimSpace(title) == "" || strings.ContainsAny(title, "\r\n\x00") || seen[name] { + return fmt.Errorf("invalid or duplicate Playwright test %q; use file :: title", name) + } + seen[name] = true + } + return nil +} + +// Playwright reports spec.file relative to its config rootDir. Nested describe +// titles form part of a test's identity, as does the single selected project. +type browserSuite struct { + Title string + Specs []struct { + Title, File string + Tests []struct { + ProjectName, ExpectedStatus, Status string + Results []struct { + Status string + Retry int + } + } + } + Suites []browserSuite +} + +func validateBrowserReport(data []byte, c Check, workspace string) error { + if err := uniqueJSONKeys(data); err != nil { + return fmt.Errorf("invalid Playwright JSON report: %w", err) + } + var report struct { + Config struct{ Version, RootDir string } + Suites []browserSuite + Errors []json.RawMessage + Stats struct{ Expected, Skipped, Unexpected, Flaky int } + } + if err := json.Unmarshal(data, &report); err != nil { + return fmt.Errorf("invalid Playwright JSON report: %w", err) + } + if report.Config.Version == "" || len(report.Errors) > 0 || report.Stats.Skipped != 0 || report.Stats.Unexpected != 0 || report.Stats.Flaky != 0 { + return fmt.Errorf("Playwright report contains errors, skipped, unexpected or flaky tests, or lacks runner identity") + } + reportRoot := "" + if workspace != "" { + rel, err := filepath.Rel(workspace, report.Config.RootDir) + if err != nil || !filepath.IsAbs(report.Config.RootDir) || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return fmt.Errorf("Playwright report root is outside the checked snapshot") + } + reportRoot = rel + } + passed := map[string]bool{} + var visit func(browserSuite, string) error + visit = func(suite browserSuite, prefix string) error { + for _, spec := range suite.Specs { + file := strings.ReplaceAll(spec.File, "\\", "/") + name := file + " :: " + prefix + spec.Title + if !browserRelativeFile(file) || strings.TrimSpace(spec.Title) == "" || len(spec.Tests) != 1 || passed[name] { + return fmt.Errorf("invalid or duplicate Playwright spec %q", name) + } + if workspace != "" { + rel := filepath.ToSlash(filepath.Join(reportRoot, filepath.FromSlash(file))) + item, err := Local(workspace, rel) + if err != nil { + return err + } + info, err := os.Stat(item) + if err != nil || !info.Mode().IsRegular() { + return fmt.Errorf("Playwright test file %q is not in the checked snapshot", rel) + } + } + test := spec.Tests[0] + if test.ProjectName != c.Project || test.ExpectedStatus != "passed" || test.Status != "expected" || len(test.Results) != 1 || test.Results[0].Status != "passed" || test.Results[0].Retry != 0 { + return fmt.Errorf("Playwright test %q did not pass on its first execution in project %q", name, c.Project) + } + passed[name] = true + } + for _, child := range suite.Suites { + if err := visit(child, prefix+child.Title+" › "); err != nil { + return err + } + } + return nil + } + for _, suite := range report.Suites { + if err := visit(suite, ""); err != nil { + return err + } + } + if len(passed) == 0 || len(passed) != report.Stats.Expected { + return fmt.Errorf("Playwright report lacks a complete passing test count") + } + for _, name := range c.Tests { + if !passed[name] { + return fmt.Errorf("required Playwright test %q did not execute and pass", name) + } + } + return nil +} + +func browserReportFile(c Check) string { return c.ID + ".playwright.json" } + +func validBrowserArtifactName(c Check, name string) bool { + if c.Kind != "playwright" { + return false + } + if name == browserReportFile(c) { + return true + } + for n := 1; n <= browserArtifactCount; n++ { + prefix := fmt.Sprintf("%s.browser-%02d", c.ID, n) + if name == prefix+".png" || name == prefix+".zip" { + return true + } + } + return false +} + +func (r ToolRunner) runBrowser(ctx context.Context, root string, c Check) Execution { + cli := os.Getenv("PDD_PLAYWRIGHT_CLI") + if !filepath.IsAbs(cli) || filepath.Base(cli) != "cli.js" { + return Execution{Status: "unavailable", ExitCode: -1, Detail: "set PDD_PLAYWRIGHT_CLI to the absolute path of a provisioned @playwright/test/cli.js"} + } + packageDir := filepath.Dir(cli) + modulesDir := filepath.Dir(filepath.Dir(packageDir)) + if filepath.Base(packageDir) != "test" || filepath.Base(filepath.Dir(packageDir)) != "@playwright" || filepath.Base(modulesDir) != "node_modules" { + return Execution{Status: "unavailable", ExitCode: -1, Detail: "PDD_PLAYWRIGHT_CLI must point to node_modules/@playwright/test/cli.js"} + } + info, err := os.Stat(cli) + if err != nil || !info.Mode().IsRegular() { + return Execution{Status: "unavailable", ExitCode: -1, Detail: "provisioned Playwright CLI file is unavailable"} + } + output, err := createScratch("pdd-browser-") + if err != nil { + return Execution{Status: "failed", ExitCode: -1, Detail: err.Error()} + } + defer removeScratch(output) + reportPath := filepath.Join(output, "report.json") + artifactsPath := filepath.Join(output, "test-results") + command := []string{"node", cli, "test", "--config=" + c.Config, "--project=" + c.Project, "--reporter=json", "--output=" + artifactsPath, "--retries=0", "--repeat-each=1", "--forbid-only", "--trace=retain-on-failure", "--no-deps"} + // Leave the separately provisioned toolchain outside the hashed source tree. + e := executeWithInput(ctx, root, command, r.Timeout, nil, "NODE_PATH="+modulesDir, "PLAYWRIGHT_JSON_OUTPUT_FILE="+reportPath, "PLAYWRIGHT_FORCE_TTY=0", "CI=1") + e.artifacts = map[string][]byte{} + data, err := readArtifact(reportPath, browserArtifactLimit) + if err == nil { + e.artifacts[browserReportFile(c)] = data + if e.Status == "passed" { + err = validateBrowserReport(data, c, root) + } + } + if e.Status == "passed" && err != nil { + e.Status = "failed" + e.Detail = "Playwright evidence: " + err.Error() + } + if e.Status != "passed" { + saveBrowserDiagnostics(artifactsPath, c.ID, e.artifacts) + } + return e +} + +// Keep only bounded browser files the user can open as screenshots or with the +// Playwright trace viewer. Fresh output lives outside the checkout snapshot. +func saveBrowserDiagnostics(root, id string, artifacts map[string][]byte) { + remaining := int64(24 << 20) + visited := 0 + _ = filepath.WalkDir(root, func(file string, entry fs.DirEntry, err error) error { + if err != nil { + return nil + } + visited++ + if visited > 1000 || len(artifacts) >= browserArtifactCount || remaining <= 0 { + return fs.SkipAll + } + if entry.IsDir() { + return nil + } + ext := path.Ext(filepath.ToSlash(file)) + if ext != ".png" && filepath.Base(file) != "trace.zip" { + return nil + } + limit := int64(browserArtifactLimit) + if remaining < limit { + limit = remaining + } + data, err := readArtifact(file, limit) + if err == nil { + artifacts[fmt.Sprintf("%s.browser-%02d%s", id, len(artifacts)+1, ext)] = data + remaining -= int64(len(data)) + } + return nil + }) +} diff --git a/internal/pdd/browser_test.go b/internal/pdd/browser_test.go new file mode 100644 index 0000000..dc9fbee --- /dev/null +++ b/internal/pdd/browser_test.go @@ -0,0 +1,128 @@ +package pdd + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +const passingBrowserReport = `{"config":{"version":"1.65.1"},"suites":[{"title":"wallet.spec.cjs","suites":[{"title":"Withdrawal","specs":[{"file":"wallet.spec.cjs","title":"denies overdraft","tests":[{"projectName":"chromium","expectedStatus":"passed","status":"expected","results":[{"status":"passed","retry":0}]}]}]}]}],"errors":[],"stats":{"expected":1,"skipped":0,"unexpected":0,"flaky":0}}` + +func TestBrowserReportRequiresNamedFirstPass(t *testing.T) { + check := Check{Kind: "playwright", Project: "chromium", Tests: []string{"wallet.spec.cjs :: Withdrawal › denies overdraft"}} + if err := validateBrowserReport([]byte(passingBrowserReport), check, ""); err != nil { + t.Fatal(err) + } + cases := map[string]string{ + "empty": "{}", + "missing": strings.Replace(passingBrowserReport, "denies overdraft", "another case", 1), + "wrong project": strings.Replace(passingBrowserReport, "chromium", "webkit", 1), + "expected failure": strings.Replace(passingBrowserReport, `"expectedStatus":"passed"`, `"expectedStatus":"failed"`, 1), + "retry": strings.Replace(passingBrowserReport, `"retry":0`, `"retry":1`, 1), + "skip": strings.Replace(passingBrowserReport, `"skipped":0`, `"skipped":1`, 1), + "flaky": strings.Replace(passingBrowserReport, `"flaky":0`, `"flaky":1`, 1), + "incomplete": strings.Replace(passingBrowserReport, `"expected":1`, `"expected":2`, 1), + "duplicate keys": strings.Replace(passingBrowserReport, `"expected":1`, `"expected":1,"expected":1`, 1), + } + for label, body := range cases { + t.Run(label, func(t *testing.T) { + if err := validateBrowserReport([]byte(body), check, ""); err == nil { + t.Fatal("accepted unsupported Playwright result") + } + }) + } +} + +type browserFixtureRunner struct { + fakeRunner + fail bool +} + +func (r *browserFixtureRunner) Run(ctx context.Context, root string, check Check) Execution { + if check.Kind != "playwright" { + return r.fakeRunner.Run(ctx, root, check) + } + e := Execution{Status: "passed", Output: "Playwright ran", artifacts: map[string][]byte{browserReportFile(check): []byte(passingBrowserReport)}} + if r.fail { + e.Status = "failed" + e.ExitCode = 1 + e.Detail = "browser assertion failed" + e.artifacts[check.ID+".browser-02.png"] = []byte("test screenshot fixture") + e.artifacts[check.ID+".browser-03.zip"] = []byte("test trace fixture") + } + return e +} + +func TestBrowserReceiptAndDashboard(t *testing.T) { + ctx := context.Background() + root, plan := fixture(t) + if err := os.WriteFile(filepath.Join(root, "playwright.config.cjs"), []byte("module.exports = {}"), 0644); err != nil { + t.Fatal(err) + } + check := Check{ID: "browser", Kind: "playwright", Config: "playwright.config.cjs", Project: "chromium", Tests: []string{"wallet.spec.cjs :: Withdrawal › denies overdraft"}} + plan.Checks = append(plan.Checks, check) + plan.Requirements = append(plan.Requirements, Requirement{ID: "overdraft", Text: "Denies overdraft", Kind: "empirical", Check: check.ID}) + savePlan(t, root, plan) + approveFixture(t, root, plan) + runner := &browserFixtureRunner{} + receipt, dir, err := Prove(ctx, root, plan, Options{Runner: runner}) + if err != nil || !receipt.Success { + t.Fatalf("prove: %+v %v", receipt.Gaps, err) + } + if _, _, err = StatusContext(ctx, root, plan, ""); err != nil { + t.Fatal(err) + } + output := filepath.Join(root, ".pdd", "results", "report.html") + report := func() string { + t.Helper() + if err := WriteDashboard(ctx, root, plan.ID(), "", output); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + return string(data) + } + page := report() + for _, want := range []string{"Checks and logs", "Negative examples", "killed", "evidenced", "playwright · empirical", "../receipts/", "browser.playwright.json", "deny-all.log"} { + if !strings.Contains(page, want) { + t.Errorf("missing dashboard evidence %q", want) + } + } + name := browserReportFile(check) + broken := []byte(strings.Replace(passingBrowserReport, "denies overdraft", "wrong test", 1)) + if err = AtomicWrite(filepath.Join(dir, name), broken); err != nil { + t.Fatal(err) + } + if _, _, err = StatusContext(ctx, root, plan, ""); err == nil { + t.Fatal("accepted an altered browser report") + } + // Even replacing the digest cannot make a report with the wrong test green. + receipt.Evidence[len(receipt.Evidence)-1].Artifacts[0].SHA256 = hashBytes(broken) + if err = WriteJSON(filepath.Join(dir, "receipt.json"), receipt); err != nil { + t.Fatal(err) + } + if _, _, err = StatusContext(ctx, root, plan, ""); err == nil { + t.Fatal("accepted a report missing planned test evidence") + } + if page = report(); !strings.Contains(page, "No current evidence") || strings.Contains(page, "browser.playwright.json") { + t.Fatal("dashboard showed damaged evidence") + } + runner.fail = true + receipt, _, err = Prove(ctx, root, plan, Options{Runner: runner}) + if err != nil || receipt.Success { + t.Fatalf("failed browser run: success=%t error=%v", receipt.Success, err) + } + if _, _, err = StatusContext(ctx, root, plan, ""); err != nil { + t.Fatal(err) + } + page = report() + for _, want := range []string{"browser assertion failed", "Failure screenshot", "Playwright trace"} { + if !strings.Contains(page, want) { + t.Errorf("missing failure evidence %q", want) + } + } +} diff --git a/internal/pdd/dashboard.go b/internal/pdd/dashboard.go index 643d381..5475a14 100644 --- a/internal/pdd/dashboard.go +++ b/internal/pdd/dashboard.go @@ -3,9 +3,11 @@ package pdd import ( "bytes" "context" + _ "embed" "encoding/json" "fmt" "html/template" + "net/url" "os/exec" "path/filepath" "strings" @@ -31,6 +33,7 @@ func WriteDashboard(ctx context.Context, root, name, policy, destination string) Workflow *WorkflowView JSON string Review *ReviewReport + Evidence *dashboardEvidence }{Draft: view, Workflow: workflow} if view.ValidationError == "" { r, e := ReportReview(root, view.Plan) @@ -39,6 +42,9 @@ func WriteDashboard(ctx context.Context, root, name, policy, destination string) } data.Review = &r } + if workflow != nil && workflow.receipt != nil { + data.Evidence = evidenceForDashboard(view.Plan, *workflow.receipt, workflow.receiptDir, destination) + } raw, err := json.MarshalIndent(view.Plan, "", " ") if err != nil { return err @@ -55,15 +61,73 @@ func WriteDashboard(ctx context.Context, root, name, policy, destination string) return AtomicWrite(destination, b.Bytes()) } -const dashboardTemplate = `PDD Go review

{{.Draft.Plan.Name}}

{{.Draft.Plan.Intent}}

Plan {{.Draft.PlanID}} · Draft {{.Draft.Digest}}

Human decisions → independent intent and guarantee reviews → human approval → implementation → machine verification → advisory gap review → PR

{{if .Draft.ValidationError}}

Before approval: {{.Draft.ValidationError}}

{{end}} -

Decision cards

{{range .Draft.Plan.Decisions}}

{{.Question}}

{{.Context}}

{{.File}} {{.Function}} · {{.ID}}{{range .Options}}

{{.ID}}: {{.Label}}{{if .Recommended}} (recommended){{end}} — {{.Consequence}}

{{end}}

Selected: {{.Selected}} · Reason: {{.Rationale}}

{{else}}

No decision cards recorded.

{{end}} -

Original intent → specification landscape

{{range .Draft.Plan.IntentMap}}
{{.UserWords}}

Requirements: {{range .Requirements}}{{.}} {{end}}

Decisions: {{range .Decisions}}{{.}} {{end}}

{{end}}{{range .Draft.Plan.Requirements}}{{end}}
RequirementGuaranteeEvidence
{{.ID}}{{.Text}}
{{.Ensures}}
{{.Kind}} · {{.Check}}
{{.File}} {{.Function}}
-{{with .Review}}

Frozen approval contract

Approval state: {{.Approval.State}} — {{.Approval.Message}}

Plan digest for explicit approval: {{.Digest}}{{range .Bundle.Sources}}

{{.File}}

{{range .Annotations}}
{{.}}
{{end}}{{if .Definitions}}
Pure specification definitions
{{printf "%s" .Definitions}}
{{end}}{{range .Bindings}}

{{.Owner}} · {{.Annotation}}

{{end}}
{{end}}{{end}} -

Trusted boundary and assumed clauses

{{range .Draft.Plan.Trust}}

{{.ID}}

{{.Assumption}}

Check: {{.Check}} · Affects: {{range .Affects}}{{.}} {{end}}

{{range .Clauses}}

{{.ID}}: {{.Text}} · If false: {{.Impact}} · Test {{.Test}}

{{end}}
{{end}} -

Given / When / Then tests

{{range .Draft.Plan.Scenarios}}

{{.ID}} · {{.Purpose}}

Given {{.Given}}
When {{.When}}
Then {{.Then}}

{{.Check}} / {{.Test}}
{{end}} -

Independent advisory reviews

{{range .Draft.Plan.PlanningReviews}}
{{.Role}}: {{.Result.Status}}

{{.Result.Summary}}

{{range .Result.Findings}}

{{.Severity}}: {{.Text}}

{{end}}Invocation {{.Invocation}} · {{.Driver}}
{{end}}

Model opinions are advisory; they do not establish formal correctness or authenticate a human reviewer.

-{{with .Workflow}}

Verification task graph

{{range .Tasks}}{{end}}
TaskDepends onEvidence status
{{.ID}} ({{.Kind}}){{range .DependsOn}}{{.}} {{end}}{{.Status}}

Machine receipt current: {{.MachineCurrent}} · successful: {{.MachineSuccess}} · ready for PR: {{.ReadyForPR}}

{{range .Gaps}}

{{.}}

{{end}}{{with .GapReview}}

Advisory gap review: {{.Result.Status}} — {{.Result.Summary}}

{{end}}

Potential redundant formal regression scenarios (manual review only): {{range .PruneCandidates}}{{.}} {{end}}

{{end}} -

Limits

{{range .Draft.Plan.NonGoals}}

{{.}}

{{end}}
Complete plan
{{.JSON}}
` +//go:embed dashboard.html +var dashboardTemplate string + +type dashboardLink struct{ Label, URL string } + +type dashboardCheck struct { + Evidence + Backend string + LogURL string + Files []dashboardLink + Tests []string +} + +type dashboardMutation struct { + MutationResult + LogURL string +} + +type dashboardEvidence struct { + Receipt + Verdicts map[string]string + Checks []dashboardCheck + NegativeExamples []dashboardMutation + ReceiptURL string +} + +func evidenceForDashboard(plan Plan, r Receipt, dir, destination string) *dashboardEvidence { + link := func(name string) string { + if name == "" || filepath.Base(name) != name { + return "" + } + rel, err := filepath.Rel(filepath.Dir(destination), filepath.Join(dir, name)) + if err != nil { + return "" + } + return (&url.URL{Path: filepath.ToSlash(rel)}).String() + } + result := &dashboardEvidence{Receipt: r, ReceiptURL: link("receipt.json"), Verdicts: map[string]string{}} + for _, v := range r.Requirements { + result.Verdicts[v.ID] = v.Status + } + checks := map[string]Check{} + for _, check := range plan.Checks { + checks[check.ID] = check + } + for _, e := range r.Evidence { + c := checks[e.Check] + item := dashboardCheck{Evidence: e, Backend: c.Kind, LogURL: link(e.Log), Tests: c.Tests} + for _, a := range e.Artifacts { + label := a.File + switch filepath.Ext(a.File) { + case ".json": + label = "Playwright JSON" + case ".png": + label = "Failure screenshot" + case ".zip": + label = "Playwright trace" + } + item.Files = append(item.Files, dashboardLink{Label: label, URL: link(a.File)}) + } + result.Checks = append(result.Checks, item) + } + for _, m := range r.Mutations { + result.NegativeExamples = append(result.NegativeExamples, dashboardMutation{MutationResult: m, LogURL: link(m.Log)}) + } + return result +} type PROptions struct { Title, Base, Head string diff --git a/internal/pdd/dashboard.html b/internal/pdd/dashboard.html new file mode 100644 index 0000000..38a33c4 --- /dev/null +++ b/internal/pdd/dashboard.html @@ -0,0 +1,132 @@ + + + + + +{{.Draft.Plan.Name}} · PDD review + + +
+
+

{{.Draft.Plan.Name}}

+

{{.Draft.Plan.Intent}}

+ Plan {{.Draft.PlanID}} · Draft {{.Draft.Digest}} +
+ +{{if .Draft.ValidationError}}

Before approval: {{.Draft.ValidationError}}

{{end}} +{{with .Evidence}} +
+

Verification

{{if .Success}}Passed{{else}}Failed{{end}}
+

Receipt validated against the source and approval when this report was generated.

+ Completed {{.FinishedAt}} · Source {{.SourceDigest}}{{if .GitCommit}} · Commit {{.GitCommit}}{{end}} + {{if .ReceiptURL}}

Open receipt JSON

{{end}} +
+{{else}} +
+

Verification

No current evidence

+ {{with .Workflow}}{{range .Gaps}}

{{.}}

{{end}}{{end}} +
+{{end}} +
+

Requirements

+ {{range .Draft.Plan.IntentMap}}
{{.UserWords}}
Requirements: {{range .Requirements}}{{.}} {{end}}
{{end}} +
+ + {{range .Draft.Plan.Requirements}} + + + {{end}} +
RequirementGuaranteeResult
{{.ID}}{{.Text}}{{if .Ensures}}
{{.Ensures}}
{{end}}{{if .File}}{{.File}} · {{.Function}}{{end}}
{{if $.Evidence}}{{with index $.Evidence.Verdicts .ID}}{{.}}{{end}}{{else}}Pending{{end}} +
{{.Kind}} · {{if $.Evidence}}{{.Check}}{{else}}{{.Check}}{{end}}
+
+{{with .Evidence}} +
+

Checks and logs

+
+ + {{range .Checks}} + + + + {{end}} +
CheckResultEvidence
{{.Check}}
{{.Backend}} · {{.Kind}}
{{.Execution.Status}}
{{.Execution.DurationMS}} ms{{if .Execution.Detail}}

{{.Execution.Detail}}

{{end}}
{{if .LogURL}}Execution log{{end}} + {{range .Files}}
{{if .URL}}{{.Label}}{{else}}{{.Label}}{{end}}{{end}} + {{if .Tests}}
Planned tests
    {{range .Tests}}
  • {{.}}
  • {{end}}
{{end}} + {{if .Dependencies}}

Depends on {{range .Dependencies}}{{.}} {{end}}

{{end}}
+
+{{if .NegativeExamples}} +
+

Negative examples

+

A killed mutation was rejected on the intended postcondition.

+
+ + {{range .NegativeExamples}}{{end}} +
MutationTargetResultEvidence
{{.ID}}{{.Requirement}}{{.Status}}{{if .Execution.Detail}}
{{.Execution.Detail}}{{end}}
{{if .LogURL}}Verifier log{{end}}
+
+{{end}} +{{end}} +{{if .Draft.Plan.Trust}} +

Trusted boundary

+ {{range .Draft.Plan.Trust}}

{{.ID}}

{{.Assumption}}

+ Check {{if $.Evidence}}{{.Check}}{{else}}{{.Check}}{{end}}{{if .Affects}} · Affects {{range .Affects}}{{.}} {{end}}{{end}} + {{range .Clauses}}

{{.ID}} {{.Text}} · If false: {{.Impact}} · Test {{.Test}}

{{end}} + {{end}} +
+{{end}} +{{if .Draft.Plan.Decisions}}

Decisions

+ {{range .Draft.Plan.Decisions}}

{{.Question}}

{{.Context}}

{{.File}} {{.Function}} · {{.ID}} + {{range .Options}}

{{.Label}}{{if .Recommended}} (recommended){{end}} · {{.Consequence}}

{{end}} + {{if .Selected}}

Selected {{.Selected}} · {{.Rationale}}

{{else}}

Awaiting decision

{{end}}
{{end}} +
{{end}} +{{if .Draft.Plan.Scenarios}}

Test scenarios

+ {{range .Draft.Plan.Scenarios}}

{{.ID}} · {{.Purpose}}

Given {{.Given}}
When {{.When}}
Then {{.Then}}

{{.Check}} / {{.Test}}{{end}} +
{{end}} +

Review and limits

+ {{with .Review}}

Frozen approval contract

Approval {{.Approval.State}}{{with .Approval.Message}} · {{.}}{{end}}

+ Plan digest {{.Digest}} + {{range .Bundle.Sources}}
{{.File}}{{range .Annotations}}
{{.}}
{{end}}{{if .Definitions}}
{{printf "%s" .Definitions}}
{{end}}{{range .Bindings}}

{{.Owner}} · {{.Annotation}}

{{end}}
{{end}} + {{end}} + {{range .Draft.Plan.PlanningReviews}}
Advisory {{.Role}} review · {{.Result.Status}}

{{.Result.Summary}}

{{range .Result.Findings}}

{{.Severity}} · {{.Text}}

{{end}}{{.Invocation}} · {{.Driver}}
{{end}} + {{with .Workflow}} + {{with .GapReview}}

Advisory gap review · {{.Result.Status}} · {{.Result.Summary}}

{{end}} + {{if $.Evidence}}{{range .Gaps}}

{{.}}

{{end}}{{end}} + {{if .PruneCandidates}}

Formal regression scenarios for manual review · {{range .PruneCandidates}}{{.}} {{end}}

{{end}} + {{end}} + {{if .Draft.Plan.NonGoals}}

Excluded guarantees

    {{range .Draft.Plan.NonGoals}}
  • {{.}}
  • {{end}}
{{end}} +
Complete plan
{{.JSON}}
+
+
+ diff --git a/internal/pdd/inputs.go b/internal/pdd/inputs.go index 5aa6cad..5a0e2f8 100644 --- a/internal/pdd/inputs.go +++ b/internal/pdd/inputs.go @@ -71,6 +71,16 @@ func validateInputs(root string, p Plan) error { if err != nil { return err } + for _, check := range p.Checks { + if check.Kind != "playwright" { + continue + } + for _, excluded := range policy.Exclude { + if filepath.Base(excluded) != "node_modules" || containsPath(excluded, check.Config) { + return fmt.Errorf("playwright inputs can exclude only provisioned node_modules outside the test configuration") + } + } + } protected, err := protectedInputs(root, p) if err != nil { return err diff --git a/internal/pdd/plan.go b/internal/pdd/plan.go index d462d6e..f69a793 100644 --- a/internal/pdd/plan.go +++ b/internal/pdd/plan.go @@ -53,10 +53,12 @@ type Requirement struct { } type Check struct { ID string `json:"id"` - Kind string `json:"kind"` // gobra or go-test; arbitrary commands are not accepted + Kind string `json:"kind"` // gobra, go-test or playwright Files []string `json:"files,omitempty"` Package string `json:"package,omitempty"` Tests []string `json:"tests,omitempty"` + Config string `json:"config,omitempty"` + Project string `json:"project,omitempty"` } type Trust struct { Affects []string `json:"affects,omitempty"` @@ -183,6 +185,9 @@ func (p Plan) Validate(root string) error { return err } checks[c.ID] = c + if c.Kind != "playwright" && (c.Config != "" || c.Project != "") { + return fmt.Errorf("%s: config and project require a playwright check", c.ID) + } switch c.Kind { case "gobra": if len(c.Files) == 0 || c.Package != "" || len(c.Tests) > 0 { @@ -243,6 +248,10 @@ func (p Plan) Validate(root string) error { } seen[t] = true } + case "playwright": + if err := validateBrowserCheck(root, c); err != nil { + return fmt.Errorf("%s: %w", c.ID, err) + } default: return fmt.Errorf("%s: unsupported check kind %q", c.ID, c.Kind) } @@ -296,8 +305,8 @@ func (p Plan) Validate(root string) error { } clauses[key] = true } else if r.Kind == "empirical" { - if c.Kind != "go-test" || r.File != "" || r.Function != "" || r.Ensures != "" { - return fmt.Errorf("%s: empirical requirement must map to go-test", r.ID) + if !isEmpirical(c) || r.File != "" || r.Function != "" || r.Ensures != "" { + return fmt.Errorf("%s: empirical requirement must map to go-test or playwright", r.ID) } } else { return fmt.Errorf("%s: kind must be formal or empirical", r.ID) @@ -335,7 +344,7 @@ func (p Plan) Validate(root string) error { if err := add(t.ID); err != nil { return err } - if strings.TrimSpace(t.Assumption) == "" || checks[t.Check].Kind != "go-test" { + if strings.TrimSpace(t.Assumption) == "" || !isEmpirical(checks[t.Check]) { return fmt.Errorf("%s: trust assumption requires empirical check", t.ID) } used[t.Check] = true diff --git a/internal/pdd/planning.go b/internal/pdd/planning.go index 9bbbaba..851fc73 100644 --- a/internal/pdd/planning.go +++ b/internal/pdd/planning.go @@ -154,7 +154,7 @@ func validatePlanning(root string, p Plan, ids map[string]bool) error { return err } c := checks[s.Check] - if c.Kind != "go-test" || !contains(c.Tests, s.Test) || strings.TrimSpace(s.Given) == "" || strings.TrimSpace(s.When) == "" || strings.TrimSpace(s.Then) == "" { + if !isEmpirical(c) || !contains(c.Tests, s.Test) || strings.TrimSpace(s.Given) == "" || strings.TrimSpace(s.When) == "" || strings.TrimSpace(s.Then) == "" { return fmt.Errorf("scenario %s needs a named empirical test and Given/When/Then", s.ID) } if s.Purpose != "boundary" && s.Purpose != "integration" && s.Purpose != "runtime" && s.Purpose != "formal-regression" { diff --git a/internal/pdd/prove.go b/internal/pdd/prove.go index f0768c1..d6c121f 100644 --- a/internal/pdd/prove.go +++ b/internal/pdd/prove.go @@ -15,13 +15,19 @@ import ( "time" ) +type EvidenceArtifact struct { + File string `json:"file"` + SHA256 string `json:"sha256"` +} + type Evidence struct { - Check string `json:"check"` - Kind string `json:"kind"` // formal or empirical - Execution Execution `json:"execution"` - Log string `json:"log"` - LogSHA256 string `json:"log_sha256"` - Dependencies []string `json:"dependencies,omitempty"` + Artifacts []EvidenceArtifact `json:"artifacts,omitempty"` + Check string `json:"check"` + Kind string `json:"kind"` // formal or empirical + Execution Execution `json:"execution"` + Log string `json:"log"` + LogSHA256 string `json:"log_sha256"` + Dependencies []string `json:"dependencies,omitempty"` } type MutationResult struct { ID string `json:"id"` @@ -166,6 +172,23 @@ func Prove(ctx context.Context, root string, p Plan, options Options) (Receipt, return name, hashBytes([]byte(e.Output)), nil } recordExecution := func(c Check, e Execution) error { + var artifacts []EvidenceArtifact + var names []string + for name := range e.artifacts { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + data := e.artifacts[name] + if !validBrowserArtifactName(c, name) || len(data) > browserArtifactLimit { + return fmt.Errorf("invalid browser artifact %s", name) + } + if err := AtomicWrite(filepath.Join(dir, name), data); err != nil { + return err + } + artifacts = append(artifacts, EvidenceArtifact{File: name, SHA256: hashBytes(data)}) + } + e.artifacts = nil executions[c.ID] = e log, hash, err := saveLog(c.ID, e) if err != nil { @@ -175,7 +198,7 @@ func Prove(ctx context.Context, root string, p Plan, options Options) (Receipt, if c.Kind == "gobra" { kind = "formal" } - receipt.Evidence = append(receipt.Evidence, Evidence{Check: c.ID, Kind: kind, Execution: e, Log: log, LogSHA256: hash, Dependencies: graph.Dependencies[c.ID]}) + receipt.Evidence = append(receipt.Evidence, Evidence{Check: c.ID, Kind: kind, Execution: e, Log: log, LogSHA256: hash, Dependencies: graph.Dependencies[c.ID], Artifacts: artifacts}) if e.Status != "passed" { receipt.Gaps = append(receipt.Gaps, c.ID+": "+e.Status+" "+e.Detail) } @@ -396,6 +419,11 @@ func Markdown(r Receipt) string { for _, e := range r.Evidence { fmt.Fprintf(&b, "| %s | %s | %s | %d ms | %s | [%s](%s) |\n", e.Check, e.Kind, e.Execution.Status, e.Execution.DurationMS, strings.Join(e.Dependencies, ", "), e.Log, e.Log) } + for _, e := range r.Evidence { + for _, a := range e.Artifacts { + fmt.Fprintf(&b, "\n- %s browser evidence: [%s](%s)\n", e.Check, a.File, a.File) + } + } fmt.Fprint(&b, "\n## Trusted boundary\n\n") for _, t := range r.Trust { fmt.Fprintf(&b, "- %s: %s (empirical check: %s)\n", t.ID, t.Assumption, t.Check) @@ -532,7 +560,7 @@ func StatusContext(ctx context.Context, root string, p Plan, policyPath string) deps := append([]string{}, e.Dependencies...) sort.Strings(deps) kind := "formal" - if c.Kind == "go-test" { + if isEmpirical(c) { kind = "empirical" } if !ok || e.Kind != kind || e.Execution.Status != "passed" || e.Execution.ExitCode != 0 || strings.Join(deps, ",") != strings.Join(graph.Dependencies[c.ID], ",") { @@ -614,8 +642,35 @@ func StatusContext(ctx context.Context, root string, p Plan, policyPath string) if err != nil { return receipt, dir, err } - if receipt.Success { - if err = validateExecutionOutput(checks[e.Check], recordedStdout(string(data)), string(data)); err != nil { + c, planned := checks[e.Check] + if !planned || len(e.Artifacts) > browserArtifactCount { + return receipt, dir, fmt.Errorf("receipt has unexpected check or artifacts") + } + var browserReport []byte + seen := map[string]bool{} + for _, artifact := range e.Artifacts { + if !validBrowserArtifactName(c, artifact.File) || seen[artifact.File] { + return receipt, dir, fmt.Errorf("invalid browser artifact %s", artifact.File) + } + seen[artifact.File] = true + attachment, err := checkLog(artifact.File, artifact.SHA256) + if err != nil || len(attachment) > browserArtifactLimit { + return receipt, dir, fmt.Errorf("missing or altered browser artifact %s", artifact.File) + } + if artifact.File == browserReportFile(c) { + browserReport = attachment + } + } + if receipt.Success || e.Execution.Status == "passed" { + if c.Kind == "playwright" { + if e.Execution.ExitCode != 0 { + return receipt, dir, fmt.Errorf("Playwright passed despite nonzero exit") + } + err = validateBrowserReport(browserReport, c, "") + } else { + err = validateExecutionOutput(c, recordedStdout(string(data)), string(data)) + } + if err != nil { return receipt, dir, fmt.Errorf("saved check %s does not support success: %w", e.Check, err) } } diff --git a/internal/pdd/runner.go b/internal/pdd/runner.go index 3fce922..1acda01 100644 --- a/internal/pdd/runner.go +++ b/internal/pdd/runner.go @@ -28,6 +28,7 @@ type Execution struct { ExitCode int `json:"exit_code"` Output string `json:"-"` stdout string + artifacts map[string][]byte Detail string `json:"detail,omitempty"` DurationMS int64 `json:"duration_ms"` } @@ -43,6 +44,9 @@ type ToolRunner struct { var formalSummaryRE = regexp.MustCompile(`(?i)Gobra found ([0-9]+) errors?\.`) func (r ToolRunner) Run(ctx context.Context, root string, c Check) Execution { + if c.Kind == "playwright" { + return r.runBrowser(ctx, root, c) + } goExe := r.Go if goExe == "" { goExe = "go" @@ -118,7 +122,7 @@ type processControl struct { func execute(ctx context.Context, root string, command []string, timeout time.Duration) Execution { return executeWithInput(ctx, root, command, timeout, nil) } -func executeWithInput(ctx context.Context, root string, command []string, timeout time.Duration, input []byte) Execution { +func executeWithInput(ctx context.Context, root string, command []string, timeout time.Duration, input []byte, env ...string) Execution { e := Execution{Status: "failed", Command: command, ExitCode: -1} if timeout <= 0 { timeout = 2 * time.Minute @@ -138,6 +142,7 @@ func executeWithInput(ctx context.Context, root string, command []string, timeou } cmd.WaitDelay = 2 * time.Second cmd.Env = append(os.Environ(), "GOWORK=off", "GOFLAGS=", "GOOS="+runtime.GOOS, "GOARCH="+runtime.GOARCH) + cmd.Env = append(cmd.Env, env...) var output, diagnostics boundedOutput cmd.Stdout = &output cmd.Stderr = &diagnostics diff --git a/internal/pdd/workflow.go b/internal/pdd/workflow.go index cfd35da..a5d7f2d 100644 --- a/internal/pdd/workflow.go +++ b/internal/pdd/workflow.go @@ -19,6 +19,8 @@ type BuildTask struct { Status string `json:"status"` } type WorkflowView struct { + receipt *Receipt + receiptDir string PlanID string `json:"plan_id"` Digest string `json:"digest"` Tasks []BuildTask `json:"tasks"` @@ -46,6 +48,7 @@ func WorkflowReport(ctx context.Context, root string, p Plan, policy string) (Wo view.MachineSuccess = statusErr == nil && receipt.Success states := map[string]string{} if statusErr == nil { + view.receipt, view.receiptDir = &receipt, dir for _, e := range receipt.Evidence { states[e.Check] = e.Execution.Status } @@ -159,7 +162,7 @@ func copyArtifactDirectory(source, destination string) error { if entry.IsDir() { continue } - data, err := readArtifact(filepath.Join(source, entry.Name()), 8<<20) + data, err := readArtifact(filepath.Join(source, entry.Name()), (8<<20)+1024) if err != nil { return err }